From 1d92666865b35341e076efbefddf6e73b5e1542e Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Wed, 7 Sep 2022 15:53:06 +0200 Subject: Add support for recoverable syntax errors --- build.rs | 6 +- src/compiler.rs | 2 +- src/construct/document.rs | 21 +- src/construct/mdx_jsx_text.rs | 88 ++- src/lib.rs | 103 ++- src/parser.rs | 8 +- src/state.rs | 25 +- src/subtokenize.rs | 8 +- src/tokenizer.rs | 17 +- tests/attention.rs | 14 +- tests/autolink.rs | 45 +- tests/block_quote.rs | 6 +- tests/character_escape.rs | 14 +- tests/character_reference.rs | 14 +- tests/code_fenced.rs | 6 +- tests/code_indented.rs | 22 +- tests/code_text.rs | 8 +- tests/commonmark.rs | 1308 +++++++++++++++++++------------------ tests/definition.rs | 13 +- tests/frontmatter.rs | 20 +- tests/gfm_autolink_literal.rs | 122 ++-- tests/gfm_footnote.rs | 103 +-- tests/gfm_strikethrough.rs | 30 +- tests/gfm_table.rs | 126 ++-- tests/gfm_tagfilter.rs | 12 +- tests/gfm_task_list_item.rs | 14 +- tests/hard_break_escape.rs | 6 +- tests/hard_break_trailing.rs | 6 +- tests/heading_atx.rs | 6 +- tests/heading_setext.rs | 6 +- tests/html_flow.rs | 328 +++++----- tests/html_text.rs | 138 ++-- tests/image.rs | 8 +- tests/link_reference.rs | 10 +- tests/link_resource.rs | 20 +- tests/list.rs | 33 +- tests/math_flow.rs | 80 +-- tests/math_text.rs | 54 +- tests/misc_bom.rs | 4 +- tests/misc_dangerous_html.rs | 6 +- tests/misc_dangerous_protocol.rs | 12 +- tests/misc_default_line_ending.rs | 8 +- tests/misc_line_ending.rs | 22 +- tests/misc_soft_break.rs | 4 +- tests/misc_tabs.rs | 14 +- tests/misc_url.rs | 4 +- tests/misc_zero.rs | 4 +- tests/text.rs | 4 +- tests/thematic_break.rs | 6 +- 49 files changed, 1550 insertions(+), 1388 deletions(-) diff --git a/build.rs b/build.rs index b485bb5..e32493c 100644 --- a/build.rs +++ b/build.rs @@ -53,7 +53,7 @@ async fn commonmark() { format!("{}\n", parts[1]) }; - let test = format!(" assert_eq!(\n micromark_with_options(\n r###\"{}\"###,\n &danger\n ),\n r###\"{}\"###,\n r###\"{} ({})\"###\n);", input, output, section, number); + let test = format!(" assert_eq!(\n micromark_with_options(\n r###\"{}\"###,\n &danger\n )?,\n r###\"{}\"###,\n r###\"{} ({})\"###\n);", input, output, section, number); cases.push(test); @@ -73,7 +73,7 @@ use pretty_assertions::assert_eq; #[rustfmt::skip] #[test] -fn commonmark() {{ +fn commonmark() -> Result<(), String> {{ let danger = Options {{ allow_dangerous_html: true, allow_dangerous_protocol: true, @@ -81,6 +81,8 @@ fn commonmark() {{ }}; {} + + Ok(()) }} ", cases.join("\n\n") diff --git a/src/compiler.rs b/src/compiler.rs index b271768..4f0f958 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -333,7 +333,7 @@ pub fn compile(events: &[Event], bytes: &[u8], options: &Options) -> String { generate_footnote_section(&mut context); } - assert_eq!(context.buffers.len(), 1, "expected 1 final buffer"); + debug_assert_eq!(context.buffers.len(), 1, "expected 1 final buffer"); context .buffers .get(0) diff --git a/src/construct/document.rs b/src/construct/document.rs index e31e58d..57c5f3a 100644 --- a/src/construct/document.rs +++ b/src/construct/document.rs @@ -14,7 +14,7 @@ use crate::state::{Name as StateName, State}; use crate::subtokenize::divide_events; use crate::tokenizer::{Container, ContainerState, Tokenizer}; use crate::util::skip; -use alloc::{boxed::Box, vec::Vec}; +use alloc::{boxed::Box, string::String, vec::Vec}; /// Phases where we can exit containers. #[derive(Debug, PartialEq)] @@ -266,7 +266,9 @@ pub fn container_new_after(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.document_continued != tokenizer.tokenize_state.document_container_stack.len() { - exit_containers(tokenizer, &Phase::Prefix); + if let Err(message) = exit_containers(tokenizer, &Phase::Prefix) { + return State::Error(message); + } } // We are “piercing” into the flow with a new container. @@ -361,6 +363,7 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { let state = tokenizer .tokenize_state .document_child_state + .take() .unwrap_or(State::Next(StateName::FlowStart)); tokenizer.tokenize_state.document_exits.push(None); @@ -439,13 +442,17 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.document_continued != tokenizer.tokenize_state.document_container_stack.len() { - exit_containers(tokenizer, &Phase::After); + if let Err(message) = exit_containers(tokenizer, &Phase::After) { + return State::Error(message); + } } match tokenizer.current { None => { tokenizer.tokenize_state.document_continued = 0; - exit_containers(tokenizer, &Phase::Eof); + if let Err(message) = exit_containers(tokenizer, &Phase::Eof) { + return State::Error(message); + } resolve(tokenizer); State::Ok } @@ -461,7 +468,7 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { } /// Close containers (and flow if needed). -fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) { +fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) -> Result<(), String> { let mut stack_close = tokenizer .tokenize_state .document_container_stack @@ -477,7 +484,7 @@ fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) { .take() .unwrap_or(State::Next(StateName::FlowStart)); - child.flush(state, false); + child.flush(state, false)?; } if !stack_close.is_empty() { @@ -524,6 +531,8 @@ fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) { } child.interrupt = false; + + Ok(()) } // Inject everything together. diff --git a/src/construct/mdx_jsx_text.rs b/src/construct/mdx_jsx_text.rs index deeb3e9..4c71fec 100644 --- a/src/construct/mdx_jsx_text.rs +++ b/src/construct/mdx_jsx_text.rs @@ -76,10 +76,10 @@ pub fn name_before(tokenizer: &mut Tokenizer) -> State { // Fragment opening tag. Some(b'>') => State::Retry(StateName::MdxJsxTextTagEnd), _ => { - // To do: unicode. - let char_opt = char_after_index(tokenizer.parse_state.bytes, tokenizer.point.index); - - if id_start(char_opt) { + if id_start(char_after_index( + tokenizer.parse_state.bytes, + tokenizer.point.index, + )) { tokenizer.enter(Name::MdxJsxTextTagName); tokenizer.enter(Name::MdxJsxTextTagNamePrimary); tokenizer.consume(); @@ -111,34 +111,32 @@ pub fn name_before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn closing_tag_name_before(tokenizer: &mut Tokenizer) -> State { - match tokenizer.current { - // Fragment closing tag. - Some(b'>') => State::Retry(StateName::MdxJsxTextTagEnd), - // Start of a closing tag name. - _ => { - // To do: unicode. - let char_opt = char_after_index(tokenizer.parse_state.bytes, tokenizer.point.index); - - if id_start(char_opt) { - tokenizer.enter(Name::MdxJsxTextTagName); - tokenizer.enter(Name::MdxJsxTextTagNamePrimary); - tokenizer.consume(); - State::Next(StateName::MdxJsxTextPrimaryName) - } else { - crash( - tokenizer, - "before name", - &format!( - "a character that can start a name, such as a letter, `$`, or `_`{}", - if tokenizer.current == Some(b'*' | b'/') { - " (note: JS comments in JSX tags are not supported in MDX)" - } else { - "" - } - ), - ) - } - } + // Fragment closing tag. + if let Some(b'>') = tokenizer.current { + State::Retry(StateName::MdxJsxTextTagEnd) + } + // Start of a closing tag name. + else if id_start(char_after_index( + tokenizer.parse_state.bytes, + tokenizer.point.index, + )) { + tokenizer.enter(Name::MdxJsxTextTagName); + tokenizer.enter(Name::MdxJsxTextTagNamePrimary); + tokenizer.consume(); + State::Next(StateName::MdxJsxTextPrimaryName) + } else { + crash( + tokenizer, + "before name", + &format!( + "a character that can start a name, such as a letter, `$`, or `_`{}", + if tokenizer.current == Some(b'*' | b'/') { + " (note: JS comments in JSX tags are not supported in MDX)" + } else { + "" + } + ), + ) } } @@ -162,7 +160,6 @@ pub fn primary_name(tokenizer: &mut Tokenizer) -> State { } // Continuation of name: remain. // Allow continuation bytes. - // To do: unicode. else if matches!(tokenizer.current, Some(0x80..=0xBF)) || id_cont(char_after_index( tokenizer.parse_state.bytes, @@ -284,7 +281,7 @@ pub fn member_name(tokenizer: &mut Tokenizer) -> State { State::Retry(StateName::MdxJsxTextEsWhitespaceStart) } // Continuation of name: remain. - // To do: unicode. + // Allow continuation bytes. else if matches!(tokenizer.current, Some(0x80..=0xBF)) || id_cont(char_after_index( tokenizer.parse_state.bytes, @@ -398,7 +395,7 @@ pub fn local_name(tokenizer: &mut Tokenizer) -> State { State::Retry(StateName::MdxJsxTextEsWhitespaceStart) } // Continuation of name: remain. - // To do: unicode. + // Allow continuation bytes. else if matches!(tokenizer.current, Some(0x80..=0xBF)) || id_cont(char_after_index( tokenizer.parse_state.bytes, @@ -516,8 +513,8 @@ pub fn attribute_primary_name(tokenizer: &mut Tokenizer) -> State { ); State::Retry(StateName::MdxJsxTextEsWhitespaceStart) } - // Continuation of the attribute name: remain. - // To do: unicode. + // Continuation of name: remain. + // Allow continuation bytes. else if matches!(tokenizer.current, Some(0x80..=0xBF)) || id_cont(char_after_index( tokenizer.parse_state.bytes, @@ -525,7 +522,7 @@ pub fn attribute_primary_name(tokenizer: &mut Tokenizer) -> State { )) { tokenizer.consume(); - State::Next(StateName::MdxJsxTextLocalName) + State::Next(StateName::MdxJsxTextAttributePrimaryName) } else { crash( tokenizer, @@ -643,8 +640,8 @@ pub fn attribute_local_name(tokenizer: &mut Tokenizer) -> State { ); State::Retry(StateName::MdxJsxTextEsWhitespaceStart) } - // Continuation of local name: remain. - // To do: unicode. + // Continuation of name: remain. + // Allow continuation bytes. else if matches!(tokenizer.current, Some(0x80..=0xBF)) || id_cont(char_after_index( tokenizer.parse_state.bytes, @@ -906,7 +903,6 @@ pub fn es_whitespace_inside(tokenizer: &mut Tokenizer) -> State { } } -// To do: unicode. fn id_start(code: Option) -> bool { if let Some(char) = code { UnicodeID::is_id_start(char) || matches!(char, '$' | '_') @@ -915,7 +911,6 @@ fn id_start(code: Option) -> bool { } } -// To do: unicode. fn id_cont(code: Option) -> bool { if let Some(char) = code { UnicodeID::is_id_continue(char) || matches!(char, '-' | '\u{200c}' | '\u{200d}') @@ -924,25 +919,24 @@ fn id_cont(code: Option) -> bool { } } -fn crash(tokenizer: &Tokenizer, at: &str, expect: &str) -> ! { +fn crash(tokenizer: &Tokenizer, at: &str, expect: &str) -> State { // To do: externalize this, and the print mechanism in the tokenizer, // to one proper formatter. - // To do: figure out how Rust does errors? let actual = match tokenizer.current { None => "end of file".to_string(), Some(byte) => format_byte(byte), }; - unreachable!( + State::Error(format!( "{}:{}: Unexpected {} {}, expected {}", tokenizer.point.line, tokenizer.point.column, actual, at, expect - ) + )) } fn format_byte(byte: u8) -> String { match byte { b'`' => "`` ` ``".to_string(), b' '..=b'~' => format!("`{}`", str::from_utf8(&[byte]).unwrap()), - _ => format!("U+{:>04X}", byte), + _ => format!("character U+{:>04X}", byte), } } diff --git a/src/lib.rs b/src/lib.rs index 7fd705b..e0b6da2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -406,6 +406,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options}; + /// # fn main() -> Result<(), String> { /// /// // micromark is safe by default: /// assert_eq!( @@ -421,9 +422,11 @@ pub struct Options { /// allow_dangerous_html: true, /// ..Options::default() /// } - /// ), + /// )?, /// "

Hi, venus!

" /// ); + /// # Ok(()) + /// # } /// ``` pub allow_dangerous_html: bool, @@ -435,6 +438,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options}; + /// # fn main() -> Result<(), String> { /// /// // micromark is safe by default: /// assert_eq!( @@ -450,9 +454,11 @@ pub struct Options { /// allow_dangerous_protocol: true, /// ..Options::default() /// } - /// ), + /// )?, /// "

javascript:alert(1)

" /// ); + /// # Ok(()) + /// # } /// ``` pub allow_dangerous_protocol: bool, @@ -463,6 +469,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // micromark follows CommonMark by default: /// assert_eq!( @@ -481,9 +488,11 @@ pub struct Options { /// }, /// ..Options::default() /// } - /// ), + /// )?, /// "

indented code?

" /// ); + /// # Ok(()) + /// # } /// ``` pub constructs: Constructs, @@ -503,6 +512,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, LineEnding}; + /// # fn main() -> Result<(), String> { /// /// // micromark uses `\n` by default: /// assert_eq!( @@ -518,9 +528,11 @@ pub struct Options { /// default_line_ending: LineEnding::CarriageReturnLineFeed, /// ..Options::default() /// } - /// ), + /// )?, /// "
\r\n

a

\r\n
" /// ); + /// # Ok(()) + /// # } /// ``` pub default_line_ending: LineEnding, @@ -534,6 +546,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // `"Footnotes"` is used by default: /// assert_eq!( @@ -543,7 +556,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); /// @@ -556,9 +569,11 @@ pub struct Options { /// gfm_footnote_label: Some("Notes de bas de page".to_string()), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Notes de bas de page

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_footnote_label: Option, @@ -570,6 +585,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // `"h2"` is used by default: /// assert_eq!( @@ -579,7 +595,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); /// @@ -592,9 +608,11 @@ pub struct Options { /// gfm_footnote_label_tag_name: Some("h1".to_string()), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_footnote_label_tag_name: Option, @@ -612,6 +630,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // `"class=\"sr-only\""` is used by default: /// assert_eq!( @@ -621,7 +640,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); /// @@ -634,9 +653,11 @@ pub struct Options { /// gfm_footnote_label_attributes: Some("class=\"footnote-heading\"".to_string()), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_footnote_label_attributes: Option, @@ -649,6 +670,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // `"Back to content"` is used by default: /// assert_eq!( @@ -658,7 +680,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); /// @@ -671,9 +693,11 @@ pub struct Options { /// gfm_footnote_back_label: Some("Arrière".to_string()), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_footnote_back_label: Option, @@ -696,6 +720,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // `"user-content-"` is used by default: /// assert_eq!( @@ -705,7 +730,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); /// @@ -718,9 +743,11 @@ pub struct Options { /// gfm_footnote_clobber_prefix: Some("".to_string()), /// ..Options::default() /// } - /// ), + /// )?, /// "

1

\n

Footnotes

\n
    \n
  1. \n

    b ↩

    \n
    2. \n
\n
\n" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_footnote_clobber_prefix: Option, @@ -733,6 +760,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark, micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // micromark supports single tildes by default: /// assert_eq!( @@ -742,7 +770,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "

a

" /// ); /// @@ -755,9 +783,11 @@ pub struct Options { /// gfm_strikethrough_single_tilde: false, /// ..Options::default() /// } - /// ), + /// )?, /// "

~a~

" /// ); + /// # Ok(()) + /// # } /// ``` pub gfm_strikethrough_single_tilde: bool, @@ -772,6 +802,7 @@ pub struct Options { /// /// ``` /// use micromark::{micromark_with_options, Options, Constructs}; + /// # fn main() -> Result<(), String> { /// /// // With `allow_dangerous_html`, micromark passes HTML through untouched: /// assert_eq!( @@ -782,7 +813,7 @@ pub struct Options { /// constructs: Constructs::gfm(), /// ..Options::default() /// } - /// ), + /// )?, /// "