Add support for parsing MDX ESM, expressions

This commit adds support for hooks that lets a user integrate another
parser with `micromark-rs`, to parse ESM and expressions according to
a particular grammar (such as a programming language, typically
JavaScript).

For an example integrating with SWC, see `tests/test_utils/mod.rs`.

The integration occurs with two functions passed in `options`:
`mdx_expression_parse` and `mdx_esm_parse`.
The can signal back to micromark when they are successful,
whether there is an error at the end (in which case micromark will
try to parse more), or whether there is a syntax error (in which case
micromark will crash).

1 files changed, 241 insertions, 0 deletions

diff --git a/tests/mdx_esm.rs b/tests/mdx_esm.rs
new file mode 100644
index 0000000..f1ea122
--- /dev/null
+++ b/tests/mdx_esm.rs
@@ -0,0 +1,241 @@
+extern crate micromark;
+mod test_utils;
+use micromark::{micromark_with_options, Constructs, Options};
+use pretty_assertions::assert_eq;
+use test_utils::{parse_esm, parse_expression};
+
+#[test]
+fn mdx_esm() -> Result<(), String> {
+    let swc = Options {
+        constructs: Constructs::mdx(),
+        mdx_esm_parse: Some(Box::new(parse_esm)),
+        mdx_expression_parse: Some(Box::new(parse_expression)),
+        ..Options::default()
+    };
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support an import"
+    );
+
+    assert_eq!(
+        micromark_with_options("export default a\n\nb", &swc)?,
+        "<p>b</p>",
+        "should support an export"
+    );
+
+    assert_eq!(
+        micromark_with_options("impossible", &swc)?,
+        "<p>impossible</p>",
+        "should not support other keywords (`impossible`)"
+    );
+
+    assert_eq!(
+        micromark_with_options("exporting", &swc)?,
+        "<p>exporting</p>",
+        "should not support other keywords (`exporting`)"
+    );
+
+    assert_eq!(
+        micromark_with_options("import.", &swc)?,
+        "<p>import.</p>",
+        "should not support a non-whitespace after the keyword"
+    );
+
+    assert_eq!(
+        micromark_with_options("import('a')", &swc)?,
+        "<p>import('a')</p>",
+        "should not support a non-whitespace after the keyword (import-as-a-function)"
+    );
+
+    assert_eq!(
+        micromark_with_options("  import a from 'b'\n  export default c", &swc)?,
+        "<p>import a from 'b'\nexport default c</p>",
+        "should not support an indent"
+    );
+
+    assert_eq!(
+        micromark_with_options("- import a from 'b'\n> export default c", &swc)?,
+        "<ul>\n<li>import a from 'b'</li>\n</ul>\n<blockquote>\n<p>export default c</p>\n</blockquote>",
+        "should not support keywords in containers"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\nexport default c", &swc)?,
+        "",
+        "should support imports and exports in the same “block”"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\n\nexport default c", &swc)?,
+        "",
+        "should support imports and exports in separate “blocks”"
+    );
+
+    assert_eq!(
+        micromark_with_options("a\n\nimport a from 'b'\n\nb\n\nexport default c", &swc)?,
+        "<p>a</p>\n<p>b</p>\n",
+        "should support imports and exports in between other constructs"
+    );
+
+    assert_eq!(
+        micromark_with_options("a\nimport a from 'b'\n\nb\nexport default c", &swc)?,
+        "<p>a\nimport a from 'b'</p>\n<p>b\nexport default c</p>",
+        "should not support import/exports when interrupting paragraphs"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a", &swc).err().unwrap(),
+        "1:9: Could not parse esm with swc: Expected ',', got '<eof>'",
+        "should crash on invalid import/exports (1)"
+    );
+
+    assert_eq!(
+        micromark_with_options("import 1/1", &swc).err().unwrap(),
+        "1:9: Could not parse esm with swc: Expected 'from', got 'numeric literal (1, 1)'",
+        "should crash on invalid import/exports (2)"
+    );
+
+    assert_eq!(
+        micromark_with_options("export {\n  a\n} from 'b'\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support line endings in import/exports"
+    );
+
+    assert_eq!(
+        micromark_with_options("export {\n\n  a\n\n} from 'b'\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support blank lines in import/exports"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\n*md*?", &swc)
+            .err()
+            .unwrap(),
+        "2:6: Could not parse esm with swc: Unexpected token `?`. Expected this, import, async, function, [ for array literal, { for object literal, @ for decorator, function, class, null, true, false, number, bigint, string, regexp, ` for template literal, (, or an identifier",
+        "should crash on markdown after import/export w/o blank line"
+    );
+
+    assert_eq!(
+        micromark_with_options("export var a = 1\n// b\n/* c */\n\nd", &swc)?,
+        "<p>d</p>",
+        "should support comments in “blocks”"
+    );
+
+    assert_eq!(
+        micromark_with_options("export var a = 1\nvar b\n\nc", &swc)
+            .err()
+            .unwrap(),
+        "2:1: Unexpected statement in code: only import/exports are supported",
+        "should crash on other statements in “blocks”"
+    );
+
+    assert_eq!(
+        micromark_with_options("import ('a')\n\nb", &swc)
+            .err()
+            .unwrap(),
+        "1:1: Unexpected statement in code: only import/exports are supported",
+        "should crash on import-as-a-function with a space `import (x)`"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\nexport {a}\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support a reexport from another import"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b';\nexport {a};\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support a reexport from another import w/ semicolons"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\nexport {a as default}\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support a reexport default from another import"
+    );
+
+    assert_eq!(
+        micromark_with_options("export var a = () => <b />", &swc)?,
+        "",
+        "should support JSX by default"
+    );
+
+    assert_eq!(
+        micromark_with_options("export {a}\n", &swc)?,
+        "",
+        "should support EOF after EOL"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\n\nexport {a}\n\nc", &swc)?,
+        "<p>c</p>",
+        "should support a reexport from another esm block (1)"
+    );
+
+    assert_eq!(
+        micromark_with_options("import a from 'b'\n\nexport {a}\n\n# c", &swc)?,
+        "<h1>c</h1>",
+        "should support a reexport from another esm block (2)"
+    );
+
+    let cases = vec![
+        ("default", "import a from \"b\""),
+        ("whole", "import * as a from \"b\""),
+        ("destructuring", "import {a} from \"b\""),
+        ("destructuring and rename", "import {a as b} from \"c\""),
+        ("default and destructuring", "import a, {b as c} from \"d\""),
+        ("default and whole", "import a, * as b from \"c\""),
+        ("side-effects", "import \"a\""),
+    ];
+
+    for case in cases {
+        assert_eq!(
+            micromark_with_options(case.1, &swc)?,
+            "",
+            "should support imports: {}",
+            case.0
+        );
+    }
+
+    let cases = vec![
+        ("var", "export var a = \"\""),
+        ("const", "export const a = \"\""),
+        ("let", "export let a = \"\""),
+        ("multiple", "export var a, b"),
+        ("multiple w/ assignment", "export var a = \"a\", b = \"b\""),
+        ("function", "export function a() {}"),
+        ("class", "export class a {}"),
+        ("destructuring", "export var {a} = {}"),
+        ("rename destructuring", "export var {a: b} = {}"),
+        ("array destructuring", "export var [a] = []"),
+        ("default", "export default a = 1"),
+        ("default function", "export default function a() {}"),
+        ("default class", "export default class a {}"),
+        ("aggregate", "export * from \"a\""),
+        ("whole reexport", "export * as a from \"b\""),
+        ("reexport destructuring", "export {a} from \"b\""),
+        (
+            "reexport destructuring w rename",
+            "export {a as b} from \"c\"",
+        ),
+        ("reexport as a default whole", "export {default} from \"b\""),
+        (
+            "reexport default and non-default",
+            "export {default as a, b} from \"c\"",
+        ),
+    ];
+
+    for case in cases {
+        assert_eq!(
+            micromark_with_options(case.1, &swc)?,
+            "",
+            "should support exports: {}",
+            case.0
+        );
+    }
+
+    Ok(())
+}