Add support for trimming whitespace around string, text

This commit introduces trimming initial and final whitespace around the
whole string or text, or around line endings inside that string or text.

*   Add `register_resolver_before`, to run resolvers earlier than others,
    used for labels
*   Add resolver to merge `data` events, which are the most frequent token
    that occurs, and can happen adjacently.
    In `micromark-js` this sped up parsing a lot
*   Fix a bug where a virtual space was not seen as an okay event
*   Refactor to enable all turned off whitespace tests

13 files changed, 120 insertions, 142 deletions

diff --git a/tests/code_indented.rs b/tests/code_indented.rs
index a7afb21..0190497 100644
--- a/tests/code_indented.rs
+++ b/tests/code_indented.rs
@@ -40,12 +40,11 @@ fn code_indented() {
         "should support blank lines in indented code (3)"
     );
 
-    // To do: trimming paragraphs.
-    // assert_eq!(
-    //     micromark("Foo\n    bar"),
-    //     "<p>Foo\nbar</p>",
-    //     "should not support interrupting paragraphs"
-    // );
+    assert_eq!(
+        micromark("Foo\n    bar"),
+        "<p>Foo\nbar</p>",
+        "should not support interrupting paragraphs"
+    );
 
     assert_eq!(
         micromark("    foo\nbar"),
diff --git a/tests/definition.rs b/tests/definition.rs
index 3edf687..ba4e384 100644
--- a/tests/definition.rs
+++ b/tests/definition.rs
@@ -382,12 +382,11 @@ fn definition() {
     //     "should support subsequent indented definitions"
     // );
 
-    // To do: trim whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("[a\n  b]: c\n\n[a\n  b]"),
-    //     "<p><a href=\"c\">a\nb</a></p>",
-    //     "should support line prefixes in definition labels"
-    // );
+    assert_eq!(
+        micromark("[a\n  b]: c\n\n[a\n  b]"),
+        "<p><a href=\"c\">a\nb</a></p>",
+        "should support line prefixes in definition labels"
+    );
 
     assert_eq!(
         micromark("[a]: )\n\n[a]"),
diff --git a/tests/hard_break_escape.rs b/tests/hard_break_escape.rs
index 2e3a3ba..c4f6f1d 100644
--- a/tests/hard_break_escape.rs
+++ b/tests/hard_break_escape.rs
@@ -9,12 +9,11 @@ fn hard_break_escape() {
         "should support a backslash to form a hard break"
     );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("foo\\\n     bar"),
-    //     "<p>foo<br />\nbar</p>",
-    //     "should support leading spaces after an escape hard break"
-    // );
+    assert_eq!(
+        micromark("foo\\\n     bar"),
+        "<p>foo<br />\nbar</p>",
+        "should support leading spaces after an escape hard break"
+    );
 
     // To do: attention.
     // assert_eq!(
diff --git a/tests/hard_break_trailing.rs b/tests/hard_break_trailing.rs
index 6c29020..0dbbbdb 100644
--- a/tests/hard_break_trailing.rs
+++ b/tests/hard_break_trailing.rs
@@ -15,12 +15,11 @@ fn hard_break_trailing() {
         "should support multiple trailing spaces"
     );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("foo  \n     bar"),
-    //     "<p>foo<br />\nbar</p>",
-    //     "should support leading spaces after a trailing hard break"
-    // );
+    assert_eq!(
+        micromark("foo  \n     bar"),
+        "<p>foo<br />\nbar</p>",
+        "should support leading spaces after a trailing hard break"
+    );
 
     // To do: attention.
     // assert_eq!(
@@ -42,12 +41,11 @@ fn hard_break_trailing() {
         "should not support trailing hard breaks in code"
     );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("foo  "),
-    //     "<p>foo</p>",
-    //     "should not support trailing hard breaks at the end of a paragraph"
-    // );
+    assert_eq!(
+        micromark("foo  "),
+        "<p>foo</p>",
+        "should not support trailing hard breaks at the end of a paragraph"
+    );
 
     assert_eq!(
         micromark("### foo  "),
@@ -55,26 +53,23 @@ fn hard_break_trailing() {
         "should not support trailing hard breaks at the end of a heading"
     );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("aaa  \t\nbb"),
-    //     "<p>aaa\nbb</p>",
-    //     "should support a mixed line suffix (1)"
-    // );
+    assert_eq!(
+        micromark("aaa  \t\nbb"),
+        "<p>aaa\nbb</p>",
+        "should support a mixed line suffix (1)"
+    );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("aaa\t  \nbb"),
-    //     "<p>aaa\nbb</p>",
-    //     "should support a mixed line suffix (2)"
-    // );
+    assert_eq!(
+        micromark("aaa\t  \nbb"),
+        "<p>aaa\nbb</p>",
+        "should support a mixed line suffix (2)"
+    );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("aaa  \t  \nbb"),
-    //     "<p>aaa\nbb</p>",
-    //     "should support a mixed line suffix (3)"
-    // );
+    assert_eq!(
+        micromark("aaa  \t  \nbb"),
+        "<p>aaa\nbb</p>",
+        "should support a mixed line suffix (3)"
+    );
 
     assert_eq!(
         micromark("aaa\0  \nbb"),
@@ -82,12 +77,11 @@ fn hard_break_trailing() {
         "should support a hard break after a replacement character"
     );
 
-    // To do: trimming whitespace in paragraphs.
-    // assert_eq!(
-    //     micromark("aaa\0\t\nbb"),
-    //     "<p>aaa�\nbb</p>",
-    //     "should support a line suffix after a replacement character"
-    // );
+    assert_eq!(
+        micromark("aaa\0\t\nbb"),
+        "<p>aaa�\nbb</p>",
+        "should support a line suffix after a replacement character"
+    );
 
     // To do: attention.
     // assert_eq!(
@@ -96,28 +90,28 @@ fn hard_break_trailing() {
     //     "should support a hard break after a span"
     // );
 
-    // To do: attention, trimming whitespace in paragraphs.
+    // To do: attention.
     // assert_eq!(
     //     micromark("*a*\t\nbb"),
     //     "<p><em>a</em>\nbb</p>",
     //     "should support a line suffix after a span"
     // );
 
-    // To do: attention, trimming whitespace in paragraphs.
+    // To do: attention.
     // assert_eq!(
     //     micromark("*a*  \t\nbb"),
     //     "<p><em>a</em>\nbb</p>",
     //     "should support a mixed line suffix after a span (1)"
     // );
 
-    // To do: attention, trimming whitespace in paragraphs.
+    // To do: attention.
     // assert_eq!(
     //     micromark("*a*\t  \nbb"),
     //     "<p><em>a</em>\nbb</p>",
     //     "should support a mixed line suffix after a span (2)"
     // );
 
-    // To do: attention, trimming whitespace in paragraphs.
+    // To do: attention.
     // assert_eq!(
     //     micromark("*a*  \t  \nbb"),
     //     "<p><em>a</em>\nbb</p>",
diff --git a/tests/heading_atx.rs b/tests/heading_atx.rs
index 67351ec..2548056 100644
--- a/tests/heading_atx.rs
+++ b/tests/heading_atx.rs
@@ -99,12 +99,11 @@ fn heading_atx() {
         "should not support four initial spaces"
     );
 
-    // To do: trimming paragraphs.
-    // assert_eq!(
-    //     micromark("foo\n    # bar"),
-    //     "<p>foo\n# bar</p>",
-    //     "should not support four initial spaces when interrupting"
-    // );
+    assert_eq!(
+        micromark("foo\n    # bar"),
+        "<p>foo\n# bar</p>",
+        "should not support four initial spaces when interrupting"
+    );
 
     assert_eq!(
         micromark("## foo ##"),
diff --git a/tests/heading_setext.rs b/tests/heading_setext.rs
index ecf22a8..e7ee9ff 100644
--- a/tests/heading_setext.rs
+++ b/tests/heading_setext.rs
@@ -24,7 +24,7 @@ fn heading_setext() {
     //     "should support line endings in setext headings"
     // );
 
-    // To do: attention, trim.
+    // To do: attention.
     // assert_eq!(
     //     micromark("  Foo *bar\nbaz*\t\n===="),
     //     "<h1>Foo <em>bar\nbaz</em></h1>",
@@ -85,19 +85,17 @@ fn heading_setext() {
         "should support whitespace before underline"
     );
 
-    // To do: trim paragraphs.
-    // assert_eq!(
-    //     micromark("Foo\n    ="),
-    //     "<p>Foo\n=</p>",
-    //     "should not support too much whitespace before underline (1)"
-    // );
+    assert_eq!(
+        micromark("Foo\n    ="),
+        "<p>Foo\n=</p>",
+        "should not support too much whitespace before underline (1)"
+    );
 
-    // To do: trim paragraphs.
-    // assert_eq!(
-    //     micromark("Foo\n\t="),
-    //     "<p>Foo\n=</p>",
-    //     "should not support too much whitespace before underline (2)"
-    // );
+    assert_eq!(
+        micromark("Foo\n\t="),
+        "<p>Foo\n=</p>",
+        "should not support too much whitespace before underline (2)"
+    );
 
     assert_eq!(
         micromark("Foo\n= ="),
@@ -111,12 +109,11 @@ fn heading_setext() {
         "should not support whitespace in the underline (2)"
     );
 
-    // To do: trim setext.
-    // assert_eq!(
-    //     micromark("Foo  \n-----"),
-    //     "<h2>Foo</h2>",
-    //     "should not support a hard break w/ spaces at the end"
-    // );
+    assert_eq!(
+        micromark("Foo  \n-----"),
+        "<h2>Foo</h2>",
+        "should not support a hard break w/ spaces at the end"
+    );
 
     assert_eq!(
         micromark("Foo\\\n-----"),
diff --git a/tests/html_flow.rs b/tests/html_flow.rs
index d942642..455c5b8 100644
--- a/tests/html_flow.rs
+++ b/tests/html_flow.rs
@@ -814,12 +814,11 @@ fn html_flow_7_complete() {
         "should not support a line ending directly after a tag name"
     );
 
-    // To do: trimming paragraphs.
-    // assert_eq!(
-    //     micromark_with_options("<x ", DANGER),
-    //     "<p>&lt;x</p>",
-    //     "should not support an eof after a space directly after a tag name"
-    // );
+    assert_eq!(
+        micromark_with_options("<x ", DANGER),
+        "<p>&lt;x</p>",
+        "should not support an eof after a space directly after a tag name"
+    );
 
     assert_eq!(
         micromark_with_options("<x/", DANGER),
diff --git a/tests/image.rs b/tests/image.rs
index 68b9717..6db6d75 100644
--- a/tests/image.rs
+++ b/tests/image.rs
@@ -102,12 +102,11 @@ fn image() {
         "should support case-insensitive labels"
     );
 
-    // To do: trim paragraphs.
-    // assert_eq!(
-    //     micromark("[foo]: /url \"title\"\n\n![foo] \n[]"),
-    //     "<p><img src=\"/url\" alt=\"foo\" title=\"title\" />\n[]</p>",
-    //     "should not support whitespace between sets of brackets"
-    // );
+    assert_eq!(
+        micromark("[foo]: /url \"title\"\n\n![foo] \n[]"),
+        "<p><img src=\"/url\" alt=\"foo\" title=\"title\" />\n[]</p>",
+        "should not support whitespace between sets of brackets"
+    );
 
     assert_eq!(
         micromark("[foo]: /url \"title\"\n\n![foo]"),
diff --git a/tests/link_reference.rs b/tests/link_reference.rs
index 0904995..372bea5 100644
--- a/tests/link_reference.rs
+++ b/tests/link_reference.rs
@@ -66,7 +66,7 @@ fn link_reference() {
     );
 
     assert_eq!(
-        micromark_with_options("[ref]: /uri\n\n[foo <bar attr=\"][ref]\">", &DANGER),
+        micromark_with_options("[ref]: /uri\n\n[foo <bar attr=\"][ref]\">", DANGER),
         "<p>[foo <bar attr=\"][ref]\"></p>",
         "should prefer HTML over link references"
     );
@@ -161,12 +161,11 @@ fn link_reference() {
         "should not support empty references"
     );
 
-    // To do: trimming whitespace.
-    // assert_eq!(
-    //     micromark("[\n ]: /uri\n\n[\n ]"),
-    //     "<p>[\n]: /uri</p>\n<p>[\n]</p>",
-    //     "should not support blank references"
-    // );
+    assert_eq!(
+        micromark("[\n ]: /uri\n\n[\n ]"),
+        "<p>[\n]: /uri</p>\n<p>[\n]</p>",
+        "should not support blank references"
+    );
 
     assert_eq!(
         micromark("[foo]: /url \"title\"\n\n[foo][]"),
@@ -187,12 +186,11 @@ fn link_reference() {
         "should match references to definitions case-insensitively"
     );
 
-    // To do: trimming whitespace.
-    // assert_eq!(
-    //     micromark("[foo]: /url \"title\"\n\n[foo] \n[]"),
-    //     "<p><a href=\"/url\" title=\"title\">foo</a>\n[]</p>",
-    //     "should not support whitespace between label and collaped reference"
-    // );
+    assert_eq!(
+        micromark("[foo]: /url \"title\"\n\n[foo] \n[]"),
+        "<p><a href=\"/url\" title=\"title\">foo</a>\n[]</p>",
+        "should not support whitespace between label and collaped reference"
+    );
 
     assert_eq!(
         micromark("[foo]: /url \"title\"\n\n[foo]"),
diff --git a/tests/link_resource.rs b/tests/link_resource.rs
index d75736e..7761569 100644
--- a/tests/link_resource.rs
+++ b/tests/link_resource.rs
@@ -443,12 +443,11 @@ fn link_resource() {
         "should not support 33 or more sets of parens"
     );
 
-    // To do: trim whitespace in string?
-    // assert_eq!(
-    //     micromark("[a](b \"\n c\")"),
-    //     "<p><a href=\"b\" title=\"\nc\">a</a></p>",
-    //     "should support an eol at the start of a title"
-    // );
+    assert_eq!(
+        micromark("[a](b \"\n c\")"),
+        "<p><a href=\"b\" title=\"\nc\">a</a></p>",
+        "should support an eol at the start of a title"
+    );
 
     assert_eq!(
         micromark("[a](b( \"c\")"),
diff --git a/tests/misc_soft_break.rs b/tests/misc_soft_break.rs
index 1704ec2..50dedc1 100644
--- a/tests/misc_soft_break.rs
+++ b/tests/misc_soft_break.rs
@@ -9,10 +9,9 @@ fn soft_break() {
         "should support line endings"
     );
 
-    // To do: trim whitespace.
-    // assert_eq!(
-    //     micromark("foo \n baz"),
-    //     "<p>foo\nbaz</p>",
-    //     "should trim spaces around line endings"
-    // );
+    assert_eq!(
+        micromark("foo \n baz"),
+        "<p>foo\nbaz</p>",
+        "should trim spaces around line endings"
+    );
 }
diff --git a/tests/misc_tabs.rs b/tests/misc_tabs.rs
index 568172e..e82738d 100644
--- a/tests/misc_tabs.rs
+++ b/tests/misc_tabs.rs
@@ -204,22 +204,20 @@ fn tabs_text() {
         "should support a space starting, and a tab ending, code"
     );
 
-    // To do: trim trailing whitespace.
-    // // Note: CM does not strip it in this case.
-    // // However, that should be a bug there: makes more sense to remove it like
-    // // trailing spaces.
-    // assert_eq!(
-    //     micromark("x\t\ny"),
-    //     "<p>x\ny</p>",
-    //     "should support a trailing tab at a line ending in a paragraph"
-    // );
-
-    // To do: trim trailing whitespace.
-    // assert_eq!(
-    //     micromark("x\n\ty"),
-    //     "<p>x\ny</p>",
-    //     "should support an initial tab after a line ending in a paragraph"
-    // );
+    // Note: CM does not strip it in this case.
+    // However, that should be a bug there: makes more sense to remove it like
+    // trailing spaces.
+    assert_eq!(
+        micromark("x\t\ny"),
+        "<p>x\ny</p>",
+        "should support a trailing tab at a line ending in a paragraph"
+    );
+
+    assert_eq!(
+        micromark("x\n\ty"),
+        "<p>x\ny</p>",
+        "should support an initial tab after a line ending in a paragraph"
+    );
 
     assert_eq!(
         micromark("x[\ty](z)"),
diff --git a/tests/thematic_break.rs b/tests/thematic_break.rs
index cbc84e0..e71ae22 100644
--- a/tests/thematic_break.rs
+++ b/tests/thematic_break.rs
@@ -63,12 +63,11 @@ fn thematic_break() {
         "should not support thematic breaks w/ 4 spaces"
     );
 
-    // To do: trimming paragraphs.
-    // assert_eq!(
-    //     micromark("Foo\n    ***"),
-    //     "<p>Foo\n***</p>",
-    //     "should not support thematic breaks w/ 4 spaces as paragraph continuation"
-    // );
+    assert_eq!(
+        micromark("Foo\n    ***"),
+        "<p>Foo\n***</p>",
+        "should not support thematic breaks w/ 4 spaces as paragraph continuation"
+    );
 
     assert_eq!(
         micromark("_____________________________________"),