diff options
| -rw-r--r-- | examples/lib.rs | 17 | ||||
| -rw-r--r-- | readme.md | 7 | ||||
| -rw-r--r-- | src/compiler.rs | 13 | ||||
| -rw-r--r-- | src/constant.rs | 7 | ||||
| -rw-r--r-- | src/construct/code_indented.rs | 2 | ||||
| -rw-r--r-- | src/construct/document.rs | 20 | ||||
| -rw-r--r-- | src/construct/flow.rs | 24 | ||||
| -rw-r--r-- | src/construct/frontmatter.rs | 293 | ||||
| -rw-r--r-- | src/construct/mod.rs | 2 | ||||
| -rw-r--r-- | src/construct/thematic_break.rs | 2 | ||||
| -rw-r--r-- | src/event.rs | 92 | ||||
| -rw-r--r-- | src/lib.rs | 12 | ||||
| -rw-r--r-- | src/state.rs | 24 | ||||
| -rw-r--r-- | src/tokenizer.rs | 5 | ||||
| -rw-r--r-- | tests/frontmatter.rs | 67 |
15 files changed, 563 insertions, 24 deletions
diff --git a/examples/lib.rs b/examples/lib.rs index b1869bb..94c2c58 100644 --- a/examples/lib.rs +++ b/examples/lib.rs @@ -1,5 +1,5 @@ extern crate micromark; -use micromark::{micromark, micromark_with_options, Options}; +use micromark::{micromark, micromark_with_options, Constructs, Options}; fn main() { // Turn on debugging. @@ -21,4 +21,19 @@ fn main() { } ) ); + + // Support extensions that are not in CommonMark. + println!( + "{:?}", + micromark_with_options( + "---\ntitle: Neptune\n---\nSome stuff on the moons of Neptune.", + &Options { + constructs: Constructs { + frontmatter: true, + ..Constructs::default() + }, + ..Options::default() + } + ) + ); } @@ -8,7 +8,6 @@ Crate docs are currently at ### Docs - [ ] (1) Add overview docs on how everything works -- [ ] (1) Add more examples ### Refactor @@ -41,14 +40,10 @@ Crate docs are currently at ### Extensions -The main thing here is is to figure out if folks could extend from the outside -with their own code, or if we need to maintain it all here. -Regardless, it is essential for the launch of `micromark-rs` that extensions -are theoretically or practically possible. The extensions below are listed from top to bottom from more important to less important. -- [ ] (1) frontmatter (yaml, toml) (flow) +- [x] (1) frontmatter (yaml, toml) (flow) — [`micromark-extension-frontmatter`](https://github.com/micromark/micromark-extension-frontmatter) - [ ] (3) autolink literal (GFM) (text) — [`micromark-extension-gfm-autolink-literal`](https://github.com/micromark/micromark-extension-gfm-autolink-literal) diff --git a/src/compiler.rs b/src/compiler.rs index db0df9b..a5b2bf5 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -319,6 +319,7 @@ fn enter(context: &mut CompileContext) { Name::Definition => on_enter_definition(context), Name::DefinitionDestinationString => on_enter_definition_destination_string(context), Name::Emphasis => on_enter_emphasis(context), + Name::Frontmatter => on_enter_frontmatter(context), Name::HtmlFlow => on_enter_html_flow(context), Name::HtmlText => on_enter_html_text(context), Name::Image => on_enter_image(context), @@ -361,6 +362,7 @@ fn exit(context: &mut CompileContext) { Name::DefinitionLabelString => on_exit_definition_label_string(context), Name::DefinitionTitleString => on_exit_definition_title_string(context), Name::Emphasis => on_exit_emphasis(context), + Name::Frontmatter => on_exit_frontmatter(context), Name::HardBreakEscape | Name::HardBreakTrailing => on_exit_break(context), Name::HeadingAtx => on_exit_heading_atx(context), Name::HeadingAtxSequence => on_exit_heading_atx_sequence(context), @@ -451,6 +453,11 @@ fn on_enter_emphasis(context: &mut CompileContext) { } } +/// Handle [`Enter`][Kind::Enter]:[`Frontmatter`][Name::Frontmatter]. +fn on_enter_frontmatter(context: &mut CompileContext) { + context.buffer(); +} + /// Handle [`Enter`][Kind::Enter]:[`HtmlFlow`][Name::HtmlFlow]. fn on_enter_html_flow(context: &mut CompileContext) { context.line_ending_if_needed(); @@ -908,6 +915,12 @@ fn on_exit_emphasis(context: &mut CompileContext) { } } +/// Handle [`Exit`][Kind::Exit]:[`Frontmatter`][Name::Frontmatter]. +fn on_exit_frontmatter(context: &mut CompileContext) { + context.resume(); + context.slurp_one_line_ending = true; +} + /// Handle [`Exit`][Kind::Exit]:[`HeadingAtx`][Name::HeadingAtx]. fn on_exit_heading_atx(context: &mut CompileContext) { let rank = context diff --git a/src/constant.rs b/src/constant.rs index b856fd0..45853a7 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -67,6 +67,13 @@ pub const CHARACTER_REFERENCE_NAMED_SIZE_MAX: usize = 31; /// [code_fenced]: crate::construct::code_fenced pub const CODE_FENCED_SEQUENCE_SIZE_MIN: usize = 3; +/// The number of markers needed for [frontmatter][] to form. +/// +/// Like many things in markdown, the number is `3`. +/// +/// [frontmatter]: crate::construct::frontmatter +pub const FRONTMATTER_SEQUENCE_SIZE: usize = 3; + /// The number of preceding spaces needed for a [hard break /// (trailing)][whitespace] to form. /// diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 89c5652..c5439f1 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -53,8 +53,8 @@ //! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element //! [html_pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element -use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; +use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::event::Name; use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; diff --git a/src/construct/document.rs b/src/construct/document.rs index 0cda368..2cc170d 100644 --- a/src/construct/document.rs +++ b/src/construct/document.rs @@ -58,13 +58,29 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { ))); tokenizer.attempt( - State::Next(StateName::DocumentContainerExistingBefore), - State::Next(StateName::DocumentContainerExistingBefore), + State::Next(StateName::DocumentBeforeFrontmatter), + State::Next(StateName::DocumentBeforeFrontmatter), ); State::Retry(StateName::BomStart) } +/// At optional frontmatter. +/// +/// ```markdown +/// > | --- +/// ^ +/// | title: Venus +/// | --- +/// ``` +pub fn before_frontmatter(tokenizer: &mut Tokenizer) -> State { + tokenizer.attempt( + State::Next(StateName::DocumentContainerNewBefore), + State::Next(StateName::DocumentContainerNewBefore), + ); + State::Retry(StateName::FrontmatterStart) +} + /// At optional existing containers. // /// ```markdown diff --git a/src/construct/flow.rs b/src/construct/flow.rs index 08c7891..f3c7685 100644 --- a/src/construct/flow.rs +++ b/src/construct/flow.rs @@ -35,28 +35,28 @@ use crate::tokenizer::Tokenizer; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'`' | b'~') => { + Some(b'#') => { tokenizer.attempt( State::Next(StateName::FlowAfter), State::Next(StateName::FlowBeforeParagraph), ); - State::Retry(StateName::CodeFencedStart) + State::Retry(StateName::HeadingAtxStart) } - Some(b'<') => { + Some(b'*' | b'_') => { tokenizer.attempt( State::Next(StateName::FlowAfter), State::Next(StateName::FlowBeforeParagraph), ); - State::Retry(StateName::HtmlFlowStart) + State::Retry(StateName::ThematicBreakStart) } - Some(b'#') => { + Some(b'<') => { tokenizer.attempt( State::Next(StateName::FlowAfter), State::Next(StateName::FlowBeforeParagraph), ); - State::Retry(StateName::HeadingAtxStart) + State::Retry(StateName::HtmlFlowStart) } - // Note: `-` is also used in thematic breaks, so it’s not included here. + // Note: `-` is also used in thematic breaks so it’s not included here. Some(b'=') => { tokenizer.attempt( State::Next(StateName::FlowAfter), @@ -64,22 +64,22 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { ); State::Retry(StateName::HeadingSetextStart) } - Some(b'*' | b'_') => { + Some(b'[') => { tokenizer.attempt( State::Next(StateName::FlowAfter), State::Next(StateName::FlowBeforeParagraph), ); - State::Retry(StateName::ThematicBreakStart) + State::Retry(StateName::DefinitionStart) } - Some(b'[') => { + Some(b'`' | b'~') => { tokenizer.attempt( State::Next(StateName::FlowAfter), State::Next(StateName::FlowBeforeParagraph), ); - State::Retry(StateName::DefinitionStart) + State::Retry(StateName::CodeFencedStart) } // Actual parsing: blank line? Indented code? Indented anything? - // Also includes `-` which can be a setext heading underline or a thematic break. + // Also includes `-` which can be a setext heading underline or thematic break. None | Some(b'\t' | b'\n' | b' ' | b'-') => State::Retry(StateName::FlowBlankLineBefore), // Must be a paragraph. Some(_) => { diff --git a/src/construct/frontmatter.rs b/src/construct/frontmatter.rs new file mode 100644 index 0000000..dc47bee --- /dev/null +++ b/src/construct/frontmatter.rs @@ -0,0 +1,293 @@ +//! Frontmatter occurs at the start of the document. +//! +//! ## Grammar +//! +//! Frontmatter forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): +//! +//! ```bnf +//! frontmatter ::= fence_open *( eol *byte ) eol fence_close +//! fence_open ::= sequence *space_or_tab +//! ; Restriction: markers in `sequence` must match markers in opening sequence. +//! fence_close ::= sequence *space_or_tab +//! sequence ::= 3'+' | 3'-' +//! ``` +//! +//! Frontmatter can only occur once. +//! It cannot occur in a container. +//! It must have a closing fence. +//! Like flow constructs, it must be followed by an eol (line ending) or +//! eof (end of file). +//! +//! ## Extension +//! +//! > 👉 **Note**: frontmatter is not part of `CommonMark`, so frontmatter is +//! > not enabled by default. +//! > You need to enable it manually. +//! > See [`Constructs`][constructs] for more info. +//! +//! As there is no spec for frontmatter in markdown, this extension follows how +//! YAML frontmatter works on `github.com`. +//! It also parses TOML frontmatter, just like YAML except that it uses a `+`. +//! +//! ## Recommendation +//! +//! When authoring markdown with frontmatter, it’s recommended to use YAML +//! frontmatter if possible. +//! While YAML has some warts, it works in the most places, so using it +//! guarantees the highest chance of portability. +//! +//! In certain ecosystems, other flavors are widely used. +//! For example, in the Rust ecosystem, TOML is often used. +//! In such cases, using TOML is an okay choice. +//! +//! ## Tokens +//! +//! * [`Frontmatter`][Name::Frontmatter] +//! * [`FrontmatterFence`][Name::FrontmatterFence] +//! * [`FrontmatterSequence`][Name::FrontmatterSequence] +//! * [`FrontmatterChunk`][Name::FrontmatterChunk] +//! * [`LineEnding`][Name::LineEnding] +//! * [`SpaceOrTab`][Name::SpaceOrTab] +//! +//! ## References +//! +//! * [`micromark-extension-frontmatter`](https://github.com/micromark/micromark-extension-frontmatter) +//! +//! [constructs]: crate::Constructs + +use crate::constant::FRONTMATTER_SEQUENCE_SIZE; +use crate::construct::partial_space_or_tab::space_or_tab; +use crate::event::Name; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; + +/// Start of frontmatter. +/// +/// ```markdown +/// > | --- +/// ^ +/// | title: "Venus" +/// | --- +/// ``` +pub fn start(tokenizer: &mut Tokenizer) -> State { + // Indent not allowed. + if tokenizer.parse_state.constructs.frontmatter + && matches!(tokenizer.current, Some(b'+' | b'-')) + { + tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); + tokenizer.enter(Name::Frontmatter); + tokenizer.enter(Name::FrontmatterFence); + tokenizer.enter(Name::FrontmatterSequence); + State::Retry(StateName::FrontmatterOpenSequence) + } else { + State::Nok + } +} + +/// In open sequence. +/// +/// ```markdown +/// > | --- +/// ^ +/// | title: "Venus" +/// | --- +/// ``` +pub fn open_sequence(tokenizer: &mut Tokenizer) -> State { + if tokenizer.current == Some(tokenizer.tokenize_state.marker) { + tokenizer.tokenize_state.size += 1; + tokenizer.consume(); + State::Next(StateName::FrontmatterOpenSequence) + } else if tokenizer.tokenize_state.size == FRONTMATTER_SEQUENCE_SIZE { + tokenizer.tokenize_state.size = 0; + tokenizer.exit(Name::FrontmatterSequence); + + if matches!(tokenizer.current, Some(b'\t' | b' ')) { + tokenizer.attempt(State::Next(StateName::FrontmatterOpenAfter), State::Nok); + State::Retry(space_or_tab(tokenizer)) + } else { + State::Retry(StateName::FrontmatterOpenAfter) + } + } else { + tokenizer.tokenize_state.marker = 0; + tokenizer.tokenize_state.size = 0; + State::Nok + } +} + +/// After open sequence. +/// +/// ```markdown +/// > | --- +/// ^ +/// | title: "Venus" +/// | --- +/// ``` +pub fn open_after(tokenizer: &mut Tokenizer) -> State { + if let Some(b'\n') = tokenizer.current { + tokenizer.exit(Name::FrontmatterFence); + tokenizer.enter(Name::LineEnding); + tokenizer.consume(); + tokenizer.exit(Name::LineEnding); + tokenizer.attempt( + State::Next(StateName::FrontmatterAfter), + State::Next(StateName::FrontmatterContentStart), + ); + State::Next(StateName::FrontmatterCloseStart) + } else { + tokenizer.tokenize_state.marker = 0; + State::Nok + } +} + +/// Start of close sequence. +/// +/// ```markdown +/// | --- +/// | title: "Venus" +/// > | --- +/// ^ +/// ``` +pub fn close_start(tokenizer: &mut Tokenizer) -> State { + if tokenizer.current == Some(tokenizer.tokenize_state.marker) { + tokenizer.enter(Name::FrontmatterFence); + tokenizer.enter(Name::FrontmatterSequence); + State::Retry(StateName::FrontmatterCloseSequence) + } else { + State::Nok + } +} + +/// In close sequence. +/// +/// ```markdown +/// | --- +/// | title: "Venus" +/// > | --- +/// ^ +/// ``` +pub fn close_sequence(tokenizer: &mut Tokenizer) -> State { + if tokenizer.current == Some(tokenizer.tokenize_state.marker) { + tokenizer.tokenize_state.size += 1; + tokenizer.consume(); + State::Next(StateName::FrontmatterCloseSequence) + } else if tokenizer.tokenize_state.size == FRONTMATTER_SEQUENCE_SIZE { + tokenizer.tokenize_state.size = 0; + tokenizer.exit(Name::FrontmatterSequence); + + if matches!(tokenizer.current, Some(b'\t' | b' ')) { + tokenizer.attempt(State::Next(StateName::FrontmatterCloseAfter), State::Nok); + State::Retry(space_or_tab(tokenizer)) + } else { + State::Retry(StateName::FrontmatterCloseAfter) + } + } else { + tokenizer.tokenize_state.size = 0; + State::Nok + } +} + +/// After close sequence. +/// +/// ```markdown +/// | --- +/// | title: "Venus" +/// > | --- +/// ^ +/// ``` +pub fn close_after(tokenizer: &mut Tokenizer) -> State { + match tokenizer.current { + None | Some(b'\n') => { + tokenizer.exit(Name::FrontmatterFence); + State::Ok + } + _ => State::Nok, + } +} + +/// Start of content chunk. +/// +/// ```markdown +/// | --- +/// > | title: "Venus" +/// ^ +/// | --- +/// ``` +pub fn content_start(tokenizer: &mut Tokenizer) -> State { + match tokenizer.current { + None | Some(b'\n') => State::Retry(StateName::FrontmatterContentEnd), + Some(_) => { + tokenizer.enter(Name::FrontmatterChunk); + State::Retry(StateName::FrontmatterContentInside) + } + } +} + +/// In content chunk. +/// +/// ```markdown +/// | --- +/// > | title: "Venus" +/// ^ +/// | --- +/// ``` +pub fn content_inside(tokenizer: &mut Tokenizer) -> State { + match tokenizer.current { + None | Some(b'\n') => { + tokenizer.exit(Name::FrontmatterChunk); + State::Retry(StateName::FrontmatterContentEnd) + } + Some(_) => { + tokenizer.consume(); + State::Next(StateName::FrontmatterContentInside) + } + } +} + +/// End of content chunk. +/// +/// ```markdown +/// | --- +/// > | title: "Venus" +/// ^ +/// | --- +/// ``` +pub fn content_end(tokenizer: &mut Tokenizer) -> State { + match tokenizer.current { + None => { + tokenizer.tokenize_state.marker = 0; + State::Nok + } + Some(b'\n') => { + tokenizer.enter(Name::LineEnding); + tokenizer.consume(); + tokenizer.exit(Name::LineEnding); + tokenizer.attempt( + State::Next(StateName::FrontmatterAfter), + State::Next(StateName::FrontmatterContentStart), + ); + State::Next(StateName::FrontmatterCloseStart) + } + Some(_) => unreachable!("expected eof/eol"), + } +} + +/// After frontmatter. +/// +/// ```markdown +/// | --- +/// | title: "Venus" +/// > | --- +/// ^ +/// ``` +pub fn after(tokenizer: &mut Tokenizer) -> State { + tokenizer.tokenize_state.marker = 0; + + match tokenizer.current { + None | Some(b'\n') => { + tokenizer.exit(Name::Frontmatter); + State::Ok + } + _ => State::Nok, + } +} diff --git a/src/construct/mod.rs b/src/construct/mod.rs index 5630143..1c1c6f7 100644 --- a/src/construct/mod.rs +++ b/src/construct/mod.rs @@ -40,6 +40,7 @@ //! * [code (indented)][code_indented] //! * [code (text)][code_text] //! * [definition][] +//! * [frontmatter][] //! * [hard break (escape)][hard_break_escape] //! * [heading (atx)][heading_atx] //! * [heading (setext)][heading_setext] @@ -139,6 +140,7 @@ pub mod code_text; pub mod definition; pub mod document; pub mod flow; +pub mod frontmatter; pub mod hard_break_escape; pub mod heading_atx; pub mod heading_setext; diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index 0a8ebe9..74fd961 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -56,8 +56,8 @@ //! [list-item]: crate::construct::list_item //! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-hr-element -use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN}; +use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::event::Name; use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; diff --git a/src/event.rs b/src/event.rs index 8058d64..f2f8ae1 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1832,10 +1832,98 @@ pub enum Name { /// ^ ^ ^ /// ``` ThematicBreakSequence, + + /// Whole frontmatter. + /// + /// ## Info + /// + /// * **Context**: + /// [document content][crate::construct::document] + /// * **Content model**: + /// [`FrontmatterFence`][Name::FrontmatterFence], + /// [`FrontmatterChunk`][Name::FrontmatterChunk], + /// [`LineEnding`][Name::LineEnding] + /// * **Construct**: + /// [`frontmatter`][crate::construct::frontmatter] + /// + /// ## Example + /// + /// ````markdown + /// > | --- + /// ^^^ + /// > | title: Neptune + /// ^^^^^^^^^^^^^^ + /// > | --- + /// ^^^ + /// ```` + Frontmatter, + /// Frontmatter chunk. + /// + /// ## Info + /// + /// * **Context**: + /// [`Frontmatter`][Name::Frontmatter] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`frontmatter`][crate::construct::frontmatter] + /// + /// ## Example + /// + /// ````markdown + /// | --- + /// > | title: Neptune + /// ^^^^^^^^^^^^^^ + /// | --- + /// ```` + FrontmatterChunk, + /// Frontmatter fence. + /// + /// ## Info + /// + /// * **Context**: + /// [`Frontmatter`][Name::Frontmatter] + /// * **Content model**: + /// [`FrontmatterSequence`][Name::FrontmatterSequence], + /// [`SpaceOrTab`][Name::SpaceOrTab] + /// * **Construct**: + /// [`frontmatter`][crate::construct::frontmatter] + /// + /// ## Example + /// + /// ````markdown + /// > | --- + /// ^^^ + /// | title: Neptune + /// > | --- + /// ^^^ + /// ```` + FrontmatterFence, + /// Frontmatter sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`FrontmatterFence`][Name::FrontmatterFence] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`frontmatter`][crate::construct::frontmatter] + /// + /// ## Example + /// + /// ````markdown + /// > | --- + /// ^^^ + /// | title: Neptune + /// > | --- + /// ^^^ + /// ```` + FrontmatterSequence, } /// List of void events, used to make sure everything is working well. -pub const VOID_EVENTS: [Name; 41] = [ +pub const VOID_EVENTS: [Name; 43] = [ Name::AttentionSequence, Name::AutolinkEmail, Name::AutolinkMarker, @@ -1860,6 +1948,8 @@ pub const VOID_EVENTS: [Name; 41] = [ Name::DefinitionMarker, Name::DefinitionTitleMarker, Name::EmphasisSequence, + Name::FrontmatterChunk, + Name::FrontmatterSequence, Name::HardBreakEscape, Name::HardBreakTrailing, Name::HeadingAtxSequence, @@ -150,6 +150,17 @@ pub struct Constructs { /// ^^^^^^^^^^ /// ``` pub definition: bool, + /// Frontmatter. + /// + /// ````markdown + /// > | --- + /// ^^^ + /// > | title: Neptune + /// ^^^^^^^^^^^^^^ + /// > | --- + /// ^^^ + /// ```` + pub frontmatter: bool, /// Hard break (escape). /// /// ```markdown @@ -246,6 +257,7 @@ impl Default for Constructs { code_fenced: true, code_text: true, definition: true, + frontmatter: false, hard_break_escape: true, hard_break_trailing: true, heading_atx: true, diff --git a/src/state.rs b/src/state.rs index f9cc39a..da935d1 100644 --- a/src/state.rs +++ b/src/state.rs @@ -110,6 +110,7 @@ pub enum Name { DestinationRawEscape, DocumentStart, + DocumentBeforeFrontmatter, DocumentContainerExistingBefore, DocumentContainerExistingAfter, DocumentContainerNewBefore, @@ -133,6 +134,17 @@ pub enum Name { FlowBlankLineAfter, FlowBeforeParagraph, + FrontmatterStart, + FrontmatterOpenSequence, + FrontmatterOpenAfter, + FrontmatterAfter, + FrontmatterContentStart, + FrontmatterContentInside, + FrontmatterContentEnd, + FrontmatterCloseStart, + FrontmatterCloseSequence, + FrontmatterCloseAfter, + HardBreakEscapeStart, HardBreakEscapeAfter, @@ -393,6 +405,7 @@ pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State { Name::DestinationRawEscape => construct::partial_destination::raw_escape, Name::DocumentStart => construct::document::start, + Name::DocumentBeforeFrontmatter => construct::document::before_frontmatter, Name::DocumentContainerExistingBefore => construct::document::container_existing_before, Name::DocumentContainerExistingAfter => construct::document::container_existing_after, Name::DocumentContainerNewBefore => construct::document::container_new_before, @@ -420,6 +433,17 @@ pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State { Name::FlowBlankLineAfter => construct::flow::blank_line_after, Name::FlowBeforeParagraph => construct::flow::before_paragraph, + Name::FrontmatterStart => construct::frontmatter::start, + Name::FrontmatterOpenSequence => construct::frontmatter::open_sequence, + Name::FrontmatterOpenAfter => construct::frontmatter::open_after, + Name::FrontmatterAfter => construct::frontmatter::after, + Name::FrontmatterContentStart => construct::frontmatter::content_start, + Name::FrontmatterContentInside => construct::frontmatter::content_inside, + Name::FrontmatterContentEnd => construct::frontmatter::content_end, + Name::FrontmatterCloseStart => construct::frontmatter::close_start, + Name::FrontmatterCloseSequence => construct::frontmatter::close_sequence, + Name::FrontmatterCloseAfter => construct::frontmatter::close_after, + Name::HardBreakEscapeStart => construct::hard_break_escape::start, Name::HardBreakEscapeAfter => construct::hard_break_escape::after, diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7eba194..2edab03 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -509,6 +509,11 @@ impl<'a> Tokenizer<'a> { /// Stack an attempt, moving to `ok` on [`State::Ok`][] and `nok` on /// [`State::Nok`][], reverting in both cases. pub fn check(&mut self, ok: State, nok: State) { + debug_assert_ne!( + nok, + State::Nok, + "checking w/ `State::Nok` should likely be an attempt" + ); // Always capture (and restore) when checking. // No need to capture (and restore) when `nok` is `State::Nok`, because the // parent attempt will do it. diff --git a/tests/frontmatter.rs b/tests/frontmatter.rs new file mode 100644 index 0000000..4882bf2 --- /dev/null +++ b/tests/frontmatter.rs @@ -0,0 +1,67 @@ +extern crate micromark; +use micromark::{micromark, micromark_with_options, Constructs, Options}; + +#[test] +fn frontmatter() { + let frontmatter = Options { + constructs: Constructs { + frontmatter: true, + ..Constructs::default() + }, + ..Options::default() + }; + + assert_eq!( + micromark("---\ntitle: Jupyter\n---"), + "<hr />\n<h2>title: Jupyter</h2>", + "should not support frontmatter by default" + ); + + assert_eq!( + micromark_with_options("---\ntitle: Jupyter\n---", &frontmatter), + "", + "should support frontmatter (yaml)" + ); + + assert_eq!( + micromark_with_options("+++\ntitle = \"Jupyter\"\n+++", &frontmatter), + "", + "should support frontmatter (toml)" + ); + + assert_eq!( + micromark_with_options("---\n---", &frontmatter), + "", + "should support empty frontmatter" + ); + + assert_eq!( + micromark_with_options("---\n---\n## Neptune", &frontmatter), + "<h2>Neptune</h2>", + "should support content after frontmatter" + ); + + assert_eq!( + micromark_with_options("## Neptune\n---\n---", &frontmatter), + "<h2>Neptune</h2>\n<hr />\n<hr />", + "should not support frontmatter after content" + ); + + assert_eq!( + micromark_with_options("> ---\n> ---\n> ## Neptune", &frontmatter), + "<blockquote>\n<hr />\n<hr />\n<h2>Neptune</h2>\n</blockquote>", + "should not support frontmatter in a container" + ); + + assert_eq!( + micromark_with_options("---", &frontmatter), + "<hr />", + "should not support just an opening fence" + ); + + assert_eq!( + micromark_with_options("---\ntitle: Neptune", &frontmatter), + "<hr />\n<p>title: Neptune</p>", + "should not support a missing closing fence" + ); +} |