diff options
Diffstat (limited to 'src/construct')
-rw-r--r-- | src/construct/attention.rs | 31 | ||||
-rw-r--r-- | src/construct/autolink.rs | 52 | ||||
-rw-r--r-- | src/construct/blank_line.rs | 35 | ||||
-rw-r--r-- | src/construct/block_quote.rs | 23 | ||||
-rw-r--r-- | src/construct/character_escape.rs | 19 | ||||
-rw-r--r-- | src/construct/character_reference.rs | 27 | ||||
-rw-r--r-- | src/construct/code_fenced.rs | 73 | ||||
-rw-r--r-- | src/construct/code_indented.rs | 44 | ||||
-rw-r--r-- | src/construct/code_text.rs | 32 | ||||
-rw-r--r-- | src/construct/definition.rs | 42 | ||||
-rw-r--r-- | src/construct/document.rs | 4 | ||||
-rw-r--r-- | src/construct/hard_break_escape.rs | 29 | ||||
-rw-r--r-- | src/construct/heading_atx.rs | 32 | ||||
-rw-r--r-- | src/construct/heading_setext.rs | 40 | ||||
-rw-r--r-- | src/construct/html_flow.rs | 48 | ||||
-rw-r--r-- | src/construct/html_text.rs | 35 | ||||
-rw-r--r-- | src/construct/label_end.rs | 2 | ||||
-rw-r--r-- | src/construct/list_item.rs | 2 | ||||
-rw-r--r-- | src/construct/mod.rs | 82 | ||||
-rw-r--r-- | src/construct/partial_data.rs | 2 | ||||
-rw-r--r-- | src/construct/partial_whitespace.rs | 6 |
21 files changed, 419 insertions, 241 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs index 1dc8868..21407b7 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -1,22 +1,29 @@ -//! Attention is a construct that occurs in the [text][] content type. +//! Attention (emphasis and strong) occurs in the [text][] content type. //! -//! How attention parses is too complex to explain in BNF. -//! Essentially, one or more of `*` or `_` form attention sequences. -//! Depending on the code before and after a sequence, it can open or close -//! attention. -//! When everything is parsed, we find each sequence that can close, and a -//! corresponding sequence that can open which uses the same marker. -//! If both sequences have two or more markers, strong is formed. -//! Otherwise emphasis is formed. +//! ## Grammar //! -//! Attention sequences do not, on their own, relate to anything in HTML. -//! When matched with another sequence, and two markers can be “taken” from -//! them, they together relate to the `<strong>` element in HTML. +//! Attention sequences form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): +//! +//! ```bnf +//! attention_sequence ::= 1*'*' | 1*'_' +//! ``` +//! +//! Sequences are matched together to form attention based on which character +//! they contain, and what character occurs before and after each sequence. +//! Otherwise they are turned into data. +//! +//! ## HTML +//! +//! When sequences match, and two markers can be “taken” from them, they +//! together relate to the `<strong>` element in HTML. //! When one marker can be taken, they relate to the `<em>` element. //! See [*§ 4.5.2 The `em` element*][html-em] and //! [*§ 4.5.3 The `strong` element*][html-strong] in the HTML spec for more //! info. //! +//! ## Recommendation +//! //! It is recommended to use asterisks for attention when writing markdown. //! //! There are some small differences in whether sequences can open and/or close diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index 37e21d9..9890aaf 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -1,22 +1,24 @@ -//! Autolinks are a construct that occurs in the [text][] content type. +//! Autolinks occur in the [text][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! Autolinks form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! autolink ::= '<' ( url | email ) '>' +//! autolink ::= '<' (url | email) '>' +//! +//! url ::= protocol *url_byte +//! protocol ::= ascii_alphabetic 0*31(protocol_byte) ':' +//! protocol_byte ::= '+' '-' '.' ascii_alphanumeric +//! url_byte ::= byte - ascii_control - ' ' //! -//! url ::= ascii_alphabetic 0*31( '+' '-' '.' ascii_alphanumeric ) ':' *( code - ascii_control - '\r' - '\n' - ' ') -//! email ::= 1*ascii_atext '@' domain *('.' domain) +//! email ::= 1*ascii_atext '@' email_domain *('.' email_domain) //! ; Restriction: up to (including) 63 character are allowed in each domain. -//! domain ::= ascii_alphanumeric *( ascii_alphanumeric | '-' ascii_alphanumeric ) -//! ascii_atext ::= ascii_alphanumeric | '#' .. '\'' | '*' | '+' | '-' | '/' | '=' | '?' | '^' .. '`' | '{' .. '~' -//! ``` +//! email_domain ::= ascii_alphanumeric *(ascii_alphanumeric | '-' ascii_alphanumeric) //! -//! Autolinks relate to the `<a>` element in HTML. -//! See [*§ 4.5.1 The `a` element*][html-a] in the HTML spec for more info. -//! When an email autolink is used (so, without a protocol), the string -//! `mailto:` is prepended before the email, when generating the `href` -//! attribute of the hyperlink. +//! ascii_atext ::= ascii_alphanumeric | '!' | '"' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | '-' | '/' | '=' | '?' | '^' | '_' | '`' | '{' | '|' | '}' | '~' +//! ``` //! //! The maximum allowed size of a scheme is `31` (inclusive), which is defined //! in [`AUTOLINK_SCHEME_SIZE_MAX`][autolink_scheme_size_max]. @@ -41,7 +43,7 @@ //! There are several cases where incorrect encoding of URLs would, in other //! languages, result in a parse error. //! In markdown, there are no errors, and URLs are normalized. -//! In addition, unicode characters are percent encoded +//! In addition, many characters are percent encoded //! ([`sanitize_uri`][sanitize_uri]). //! For example: //! @@ -82,6 +84,22 @@ //! <p><a href="#"></a><a href="https://example.com">https://example.com</a></p> //! ``` //! +//! ## HTML +//! +//! Autolinks relate to the `<a>` element in HTML. +//! See [*§ 4.5.1 The `a` element*][html_a] in the HTML spec for more info. +//! When an email autolink is used (so, without a protocol), the string +//! `mailto:` is prepended before the email, when generating the `href` +//! attribute of the hyperlink. +//! +//! ## Recommendation +//! +//! It is recommended to use labels ([label start link][label_start_link], +//! [label end][label_end]), either with a resource or a definition +//! ([definition][]), instead of autolinks, as those allow more characters in +//! URLs, and allow relative URLs and `www.` URLs. +//! They also allow for descriptive text to explain the URL in prose. +//! //! ## Tokens //! //! * [`Autolink`][Name::Autolink] @@ -95,11 +113,13 @@ //! * [*§ 6.4 Autolinks* in `CommonMark`](https://spec.commonmark.org/0.30/#autolinks) //! //! [text]: crate::construct::text +//! [definition]: crate::construct::definition +//! [label_start_link]: crate::construct::label_start_link //! [label_end]: crate::construct::label_end //! [autolink_scheme_size_max]: crate::constant::AUTOLINK_SCHEME_SIZE_MAX //! [autolink_domain_size_max]: crate::constant::AUTOLINK_DOMAIN_SIZE_MAX //! [sanitize_uri]: crate::util::sanitize_uri -//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element +//! [html_a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX}; use crate::event::Name; @@ -293,7 +313,7 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 0; let index = tokenizer.events.len(); tokenizer.exit(Name::AutolinkProtocol); - // Change the token type. + // Change the event name. tokenizer.events[index - 1].name = Name::AutolinkEmail; tokenizer.events[index].name = Name::AutolinkEmail; tokenizer.enter(Name::AutolinkMarker); diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index 928b8cc..5be406d 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -1,22 +1,39 @@ -//! Blank lines are a construct that occurs in the [flow][] content type. +//! Blank lines occur in the [flow][] content type. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Blank lines form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! blank_line ::= *(' ' '\t') +//! blank_line ::= *space_or_tab //! ``` //! +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). +//! //! Blank lines are sometimes needed, such as to differentiate a [paragraph][] //! from another paragraph. //! In several cases, blank lines are not needed between flow constructs, -//! such as between two [heading (atx)][heading-atx]s. +//! such as between two [heading (atx)][heading_atx]s. //! Sometimes, whether blank lines are present, changes the behavior of how -//! HTML is rendered, such as whether blank lines are present between list -//! items in a [list][list-item]. +//! HTML is rendered, such as whether blank lines are present inside or between +//! [list items][list_item]. //! More than one blank line is never needed in `CommonMark`. //! //! Because blank lines can be empty (line endings are not considered part of -//! it), and events cannot be empty, blank lines are not present as a token. +//! it), and events cannot be empty, blank lines are not present as an event. +//! +//! ## HTML +//! +//! Blank lines do not relate an element in HTML, except for the role they play +//! when inside or between [list items][list_item]. +//! +//! ## Recommendation +//! +//! It is recommended to always use a blank line between every flow construct, +//! to use blank lines (consistently) between list items as desired, and to +//! never use more than one blank line. //! //! ## Tokens //! @@ -27,8 +44,8 @@ //! * [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js) //! * [*§ 4.9 Blank lines* in `CommonMark`](https://spec.commonmark.org/0.30/#blank-lines) //! -//! [heading-atx]: crate::construct::heading_atx -//! [list-item]: crate::construct::list_item +//! [heading_atx]: crate::construct::heading_atx +//! [list_item]: crate::construct::list_item //! [paragraph]: crate::construct::paragraph //! [flow]: crate::construct::flow diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index 37726c5..8d7e227 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -1,6 +1,9 @@ -//! Block quote is a construct that occurs in the [document][] content type. +//! Block quotes occur in the [document][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! Block quotes form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! block_quote_start ::= '>' [ space_or_tab ] @@ -9,14 +12,24 @@ //! //! Further lines that are not prefixed with `block_quote_cont` cause the block //! quote to be exited, except when those lines are lazy continuation. -//! Like so many things in markdown, block quotes too, are very complex. -//! See [*§ Phase 1: block structure*][commonmark-block] for more on parsing -//! details. +//! Like so many things in markdown, block quotes too, are complex. +//! See [*§ Phase 1: block structure* in `CommonMark`][commonmark-block] for +//! more on parsing details. +//! +//! As block quote is a container, it takes several bytes from the start of the +//! line, while the rest of the line includes more containers or flow. +//! +//! ## HTML //! //! Block quote relates to the `<blockquote>` element in HTML. //! See [*§ 4.4.4 The `blockquote` element*][html-blockquote] in the HTML spec //! for more info. //! +//! ## Recommendation +//! +//! Always use a single space after a block quote marker (`>`). +//! Never use lazy continuation. +//! //! ## Tokens //! //! * [`BlockQuote`][Name::BlockQuote] diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 6dac458..438092e 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -1,7 +1,9 @@ -//! Character escapes are a construct that occurs in the [string][] and -//! [text][] content types. +//! Character escapes occur in the [string][] and [text][] content types. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Character escapes form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! character_escape ::= '\\' ascii_punctuation @@ -10,13 +12,20 @@ //! Like much of markdown, there are no “invalid” character escapes: just a //! slash, or a slash followed by anything other than an ASCII punctuation //! character, is exactly that: just a slash. -//! To escape (most) arbitrary characters, use a -//! [character reference][character_reference] instead +//! +//! To escape (almost all) arbitrary characters instead of only ASCII +//! punctuation, use a [character reference][character_reference] instead //! (as in, `&`, `{`, or say `	`). +//! //! It is also possible to escape a line ending in text with a similar //! construct: a [hard break (escape)][hard_break_escape] is a backslash followed //! by a line ending (that is part of the construct instead of ending it). //! +//! ## Recommendation +//! +//! If possible, use a character escape. +//! Otherwise, use a character reference. +//! //! ## Tokens //! //! * [`CharacterEscape`][Name::CharacterEscape] diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index 7935109..3bdc636 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -1,25 +1,27 @@ -//! Character references are a construct that occurs in the [string][] and -//! [text][] content types. +//! Character references occur in the [string][] and [text][] content types. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Character references form with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! character_reference ::= '&' (numeric | named) ';' //! //! numeric ::= '#' (hexadecimal | decimal) -//! ; Note: Limit of `6` imposed as all bigger numbers are invalid: +//! ; Note: Limit of `6` imposed, as all bigger numbers are invalid. //! hexadecimal ::= ('x' | 'X') 1*6(ascii_hexdigit) -//! ; Note: Limit of `7` imposed as all bigger numbers are invalid: +//! ; Note: Limit of `7` imposed, as all bigger numbers are invalid. //! decimal ::= 1*7(ascii_digit) -//! ; Note: Limit of `31` imposed by `CounterClockwiseContourIntegral`: +//! ; Note: Limit of `31` imposed, for `CounterClockwiseContourIntegral`. //! ; Note: Limited to any known named character reference (see `constants.rs`) //! named ::= 1*31(ascii_alphanumeric) //! ``` //! //! Like much of markdown, there are no “invalid” character references. //! However, for security reasons, several numeric character references parse -//! fine but are not rendered as their corresponding character and they are -//! instead replaced by a U+FFFD REPLACEMENT CHARACTER (`�`). +//! fine but are not rendered as their corresponding character. +//! They are instead replaced by a U+FFFD REPLACEMENT CHARACTER (`�`). //! See [`decode_numeric`][decode_numeric] for more info. //! //! To escape ASCII punctuation characters, use the terser @@ -33,13 +35,18 @@ //! //! Character references are parsed insensitive to casing. //! The casing of hexadecimal numeric character references has no effect. -//! The casing of named character references does not matter when parsing them, -//! but does affect whether they match. +//! The casing of named character references does not matter when parsing, but +//! does affect whether they match. //! Depending on the name, one or more cases are allowed, such as that `AMP` //! and `amp` are both allowed but other cases are not. //! See [`CHARACTER_REFERENCES`][character_references] for which //! names match. //! +//! ## Recommendation +//! +//! If possible, use a character escape. +//! Otherwise, use a character reference. +//! //! ## Tokens //! //! * [`CharacterReference`][Name::CharacterReference] diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 3812d44..748e38f 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -1,9 +1,12 @@ -//! Code (fenced) is a construct that occurs in the [flow][] content type. +//! Code (fenced) occurs in the [flow][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! Code (fenced) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! code_fenced ::= fence_open *( eol *code ) [ eol fence_close ] +//! code_fenced ::= fence_open *( eol *byte ) [ eol fence_close ] //! //! fence_open ::= sequence [ 1*space_or_tab info [ 1*space_or_tab meta ] ] *space_or_tab //! ; Restriction: the number of markers in the closing fence sequence must be @@ -13,41 +16,53 @@ //! ; marker in the opening fence sequence //! fence_close ::= sequence *space_or_tab //! sequence ::= 3*'`' | 3*'~' +//! ; Restriction: the `` ` `` character cannot occur in `info` if it is the marker. //! info ::= 1*text +//! ; Restriction: the `` ` `` character cannot occur in `meta` if it is the marker. //! meta ::= 1*text *( *space_or_tab 1*text ) -//! -//! ; Restriction: the `` ` `` character cannot occur in `text` if it is the -//! ; marker of the opening fence sequence. -//! text ::= code - eol - space_or_tab -//! eol ::= '\r' | '\r\n' | '\n' -//! space_or_tab ::= ' ' | '\t' -//! code ::= . ; any unicode code point (other than line endings). //! ``` //! -//! The above grammar does not show how whitespace is handled. -//! To parse code (fenced), let `X` be the number of whitespace characters +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). +//! +//! The above grammar does not show how indentation (with `space_or_tab`) of +//! each line is handled. +//! To parse code (fenced), let `x` be the number of `space_or_tab` characters //! before the opening fence sequence. //! Each line of text is then allowed (not required) to be indented with up -//! to `X` spaces or tabs, which are then ignored as an indent instead of being +//! to `x` spaces or tabs, which are then ignored as an indent instead of being //! considered as part of the code. //! This indent does not affect the closing fence. //! It can be indented up to a separate 3 spaces or tabs. //! A bigger indent makes it part of the code instead of a fence. //! -//! Code (fenced) relates to both the `<pre>` and the `<code>` elements in -//! HTML. -//! See [*§ 4.4.3 The `pre` element*][html-pre] and the [*§ 4.5.15 The `code` -//! element*][html-code] in the HTML spec for more info. +//! The `info` and `meta` parts are interpreted as the [string][] content type. +//! That means that [character escapes][character_escape] and +//! [character references][character_reference] are allowed. //! //! The optional `meta` part is ignored: it is not used when parsing or //! rendering. +//! //! The optional `info` part is used and is expected to specify the programming //! language that the code is in. //! Which value it holds depends on what your syntax highlighter supports, if //! one is used. +//! +//! In markdown, it is also possible to use [code (text)][code_text] in the +//! [text][] content type. +//! It is also possible to create code with the +//! [code (indented)][code_indented] construct. +//! +//! ## HTML +//! +//! Code (fenced) relates to both the `<pre>` and the `<code>` elements in +//! HTML. +//! See [*§ 4.4.3 The `pre` element*][html_pre] and the [*§ 4.5.15 The `code` +//! element*][html_code] in the HTML spec for more info. +//! //! The `info` is, when rendering to HTML, typically exposed as a class. //! This behavior stems from the HTML spec ([*§ 4.5.15 The `code` -//! element*][html-code]). +//! element*][html_code]). //! For example: //! //! ```markdown @@ -63,17 +78,11 @@ //! </code></pre> //! ``` //! -//! The `info` and `meta` parts are interpreted as the [string][] content type. -//! That means that [character escapes][character_escape] and -//! [character references][character_reference] are allowed. +//! ## Recommendation //! -//! In markdown, it is also possible to use [code (text)][code_text] in the -//! [text][] content type. -//! It is also possible to create code with the -//! [code (indented)][code_indented] construct. -//! That construct is less explicit, different from code (text), and has no -//! support for specifying the programming language, so it is recommended to -//! use code (fenced) instead of code (indented). +//! It is recommended to use code (fenced) instead of code (indented). +//! Code (fenced) is more explicit, similar to code (text), and has support +//! for specifying the programming language. //! //! ## Tokens //! @@ -94,12 +103,12 @@ //! [flow]: crate::construct::flow //! [string]: crate::construct::string //! [text]: crate::construct::text -//! [code_indented]: crate::construct::code_indented -//! [code_text]: crate::construct::code_text //! [character_escape]: crate::construct::character_escape //! [character_reference]: crate::construct::character_reference -//! [html-pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element -//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +//! [code_indented]: crate::construct::code_indented +//! [code_text]: crate::construct::code_text +//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +//! [html_pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index e3a5333..89c5652 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -1,30 +1,38 @@ -//! Code (indented) is a construct that occurs in the [flow][] content type. +//! Code (indented) occurs in the [flow][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! Code (indented) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! code_indented ::= indented_filled_line *( eol *( blank_line eol ) indented_filled_line ) +//! code_indented ::= filled_line *( eol *( blank_line eol ) filled_line ) //! -//! ; Restriction: at least one `code` must not be whitespace. -//! indented_filled_line ::= 4space_or_tab *code +//! ; Restriction: at least one `line` byte must be `text`. +//! filled_line ::= 4(space_or_tab) *line //! blank_line ::= *space_or_tab -//! eol ::= '\r' | '\r\n' | '\n' -//! code ::= . ; any unicode code point (other than line endings). -//! space_or_tab ::= ' ' | '\t' //! ``` //! -//! Code (indented) relates to both the `<pre>` and the `<code>` elements in -//! HTML. -//! See [*§ 4.4.3 The `pre` element*][html-pre] and the [*§ 4.5.15 The `code` -//! element*][html-code] in the HTML spec for more info. +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). //! //! In markdown, it is also possible to use [code (text)][code_text] in the //! [text][] content type. //! It is also possible to create code with the [code (fenced)][code_fenced] //! construct. -//! That construct is more explicit, more similar to code (text), and has -//! support for specifying the programming language that the code is in, so it -//! is recommended to use that instead of indented code. +//! +//! ## HTML +//! +//! Code (indented) relates to both the `<pre>` and the `<code>` elements in +//! HTML. +//! See [*§ 4.4.3 The `pre` element*][html_pre] and the [*§ 4.5.15 The `code` +//! element*][html_code] in the HTML spec for more info. +//! +//! ## Recommendation +//! +//! It is recommended to use code (fenced) instead of code (indented). +//! Code (fenced) is more explicit, similar to code (text), and has support +//! for specifying the programming language. //! //! ## Tokens //! @@ -40,10 +48,10 @@ //! //! [flow]: crate::construct::flow //! [text]: crate::construct::text -//! [code_text]: crate::construct::code_text //! [code_fenced]: crate::construct::code_fenced -//! [html-pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element -//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +//! [code_text]: crate::construct::code_text +//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +//! [html_pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index 7ebee96..413b5ee 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -1,12 +1,16 @@ -//! Code (text) is a construct that occurs in the [text][] content type. +//! Code (text) occurs in the [text][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! Code (text) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! ; Restriction: the number of markers in the closing sequence must be equal //! ; to the number of markers in the opening sequence. -//! code_text ::= sequence 1*code sequence +//! code_text ::= sequence 1*byte sequence //! +//! ; Restriction: not preceded or followed by `` ` ``. //! sequence ::= 1*'`' //! ``` //! @@ -18,15 +22,13 @@ //! Include more: `a``b` or include less: ``a`b``. //! ``` //! -//! When turning markdown into HTML, each line ending is turned into a space. -//! //! It is also possible to include just one grave accent (tick): //! //! ```markdown //! Include just one: `` ` ``. //! ``` //! -//! Sequences are “gready”, in that they cannot be preceded or succeeded by +//! Sequences are “gready”, in that they cannot be preceded or followed by //! more grave accents (ticks). //! To illustrate: //! @@ -53,17 +55,17 @@ //! if both exist and there is also a non-space in the code, are removed. //! Line endings, at that stage, are considered as spaces. //! -//! Code (text) relates to the `<code>` element in HTML. -//! See [*§ 4.5.15 The `code` element*][html-code] in the HTML spec for more -//! info. -//! //! In markdown, it is possible to create code with the //! [code (fenced)][code_fenced] or [code (indented)][code_indented] constructs //! in the [flow][] content type. -//! Compared to code (indented), fenced code is more explicit and more similar -//! to code (text), and it has support for specifying the programming language -//! that the code is in, so it is recommended to use that instead of indented -//! code. +//! +//! ## HTML +//! +//! Code (text) relates to the `<code>` element in HTML. +//! See [*§ 4.5.15 The `code` element*][html_code] in the HTML spec for more +//! info. +//! +//! When turning markdown into HTML, each line ending is turned into a space. //! //! ## Tokens //! @@ -81,7 +83,7 @@ //! [text]: crate::construct::text //! [code_indented]: crate::construct::code_indented //! [code_fenced]: crate::construct::code_fenced -//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element use crate::event::Name; use crate::state::{Name as StateName, State}; diff --git a/src/construct/definition.rs b/src/construct/definition.rs index 8f274ee..071e595 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -1,31 +1,29 @@ -//! Definition is a construct that occurs in the [flow] content type. +//! Definition occurs in the [flow] content type. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Definition forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! definition ::= label ':' [ whitespace ] destination [ whitespace title ] [ space_or_tab ] +//! definition ::= label ':' [ space_or_tab_eol ] destination [ space_or_tab_eol title ] [ space_or_tab ] //! //! ; See the `destination`, `title`, and `label` constructs for the BNF of //! ; those parts. //! ``` //! -//! See [`destination`][destination], [`label`][label], and [`title`][title] -//! for grammar, notes, and recommendations. +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). //! -//! Definitions in markdown do not, on their own, relate to anything in HTML. -//! When matched with a [label end (reference)][label_end], they together -//! relate to the `<a>` or `<img>` elements in HTML. -//! The definition forms its `href` or `src`, and optionally `title`, -//! attributes. -//! See [*§ 4.5.1 The `a` element*][html-a] and -//! [*§ 4.8.3 The `img` element*][html-img] in the HTML spec for more info. +//! See [`destination`][destination], [`label`][label], and [`title`][title] +//! for grammar, notes, and recommendations on each part. //! //! The `destination`, `label`, and `title` parts are interpreted as the //! [string][] content type. //! That means that [character escapes][character_escape] and //! [character references][character_reference] are allowed. //! -//! Definitions match to references through their label. +//! Definitions match to references through identifiers. //! To match, both labels must be equal after normalizing with //! [`normalize_identifier`][normalize_identifier]. //! One definition can match to multiple references. @@ -57,6 +55,16 @@ //! `<img>` when compiling, see //! [`sanitize_uri`][sanitize_uri]. //! +//! ## HTML +//! +//! Definitions in markdown do not, on their own, relate to anything in HTML. +//! When matched with a [label end (reference)][label_end], they together +//! relate to the `<a>` or `<img>` elements in HTML. +//! The definition forms its `href` or `src`, and optionally `title`, +//! attributes. +//! See [*§ 4.5.1 The `a` element*][html_a] and +//! [*§ 4.8.3 The `img` element*][html_img] in the HTML spec for more info. +//! //! ## Tokens //! //! * [`Definition`][Name::Definition] @@ -84,14 +92,14 @@ //! [string]: crate::construct::string //! [character_escape]: crate::construct::character_escape //! [character_reference]: crate::construct::character_reference -//! [label_end]: crate::construct::label_end //! [destination]: crate::construct::partial_destination -//! [title]: crate::construct::partial_title //! [label]: crate::construct::partial_label +//! [label_end]: crate::construct::label_end +//! [title]: crate::construct::partial_title //! [sanitize_uri]: crate::util::sanitize_uri::sanitize_uri //! [normalize_identifier]: crate::util::normalize_identifier -//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element -//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element +//! [html_a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element +//! [html_img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use crate::construct::partial_space_or_tab::space_or_tab; use crate::construct::partial_space_or_tab_eol::space_or_tab_eol; diff --git a/src/construct/document.rs b/src/construct/document.rs index 9def6c5..0cda368 100644 --- a/src/construct/document.rs +++ b/src/construct/document.rs @@ -6,7 +6,7 @@ //! The constructs found in flow are: //! //! * [Block quote][crate::construct::block_quote] -//! * [List][crate::construct::list_item] +//! * [List item][crate::construct::list_item] use crate::event::{Content, Event, Kind, Link, Name}; use crate::state::{Name as StateName, State}; @@ -409,7 +409,7 @@ fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) { } } - debug_assert!(found, "expected to find container token to exit"); + debug_assert!(found, "expected to find container event to exit"); } if let Some(ref mut list) = tokenizer.tokenize_state.document_exits[index] { diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 1fafa0b..64c909a 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -1,28 +1,33 @@ -//! Hard break (escape) is a construct that occurs in the [text][] content -//! type. +//! Hard break (escape) occurs in the [text][] content type. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Hard break (escape) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! ; Restriction: followed by a line ending (that is part of the construct +//! ; Restriction: followed by a line ending (that is part of the content //! ; instead of ending it). //! hard_break_escape ::= '\\' //! ``` //! -//! Hard breaks in markdown relate to the HTML element `<br>`. -//! See [*§ 4.5.27 The `br` element* in the HTML spec][html] for more info. -//! //! It is also possible to create a hard break with a //! [hard break (trailing)][hard_break_trailing]. -//! That construct is not recommended because trailing spaces are typically -//! invisible in editors, or even automatically removed, making them hard to -//! use. //! -//! It is also possible to escape punctuation characters with a similar +//! Punctuation characters can be escaped with a similar //! construct: a [character escape][character_escape] is a backslash followed //! by an ASCII punctuation character. //! Arbitrary characters can be escaped with -//! [character reference][character_reference]s. +//! [character references][character_reference]. +//! +//! ## HTML +//! +//! Hard breaks in markdown relate to the HTML element `<br>`. +//! See [*§ 4.5.27 The `br` element* in the HTML spec][html] for more info. +//! +//! ## Recommendation +//! +//! Always use hard break (escape), never hard break (trailing). //! //! ## Tokens //! diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index 3bcff54..960ae32 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -1,17 +1,16 @@ -//! Heading (atx) is a construct that occurs in the [flow] content type. +//! Heading (atx) occurs in the [flow][] content type. //! -//! They’re formed with the following BNF: +//! ## Grammar //! -//! ```bnf -//! heading_atx ::= 1*6'#' [ 1*space_or_tab text [ 1*space_or_tab 1*'#' ] ] *space_or_tab +//! Heading (atx) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! -//! text ::= code - eol -//! space_or_tab ::= ' ' | '\t' +//! ```bnf +//! heading_atx ::= 1*6'#' [ 1*space_or_tab line [ 1*space_or_tab 1*'#' ] ] *space_or_tab //! ``` //! -//! Headings in markdown relate to the `<h1>` through `<h6>` elements in HTML. -//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the -//! HTML spec][html] for more info. +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). //! //! `CommonMark` introduced the requirement on whitespace existing after the //! opening sequence and before text. @@ -25,16 +24,25 @@ //! [hard break (escape)][hard_break_escape]). //! However, their limit is that they cannot form `<h3>` through `<h6>` //! headings. -//! Due to this limitation, it is recommended to use atx headings. //! //! > 🏛 **Background**: the word *setext* originates from a small markup //! > language by Ian Feldman from 1991. -//! > See [*§ Setext* on Wikipedia][wiki-setext] for more info. +//! > See [*§ Setext* on Wikipedia][wiki_setext] for more info. //! > The word *atx* originates from a tiny markup language by Aaron Swartz //! > from 2002. //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for //! > more info. //! +//! ## HTML +//! +//! Headings in markdown relate to the `<h1>` through `<h6>` elements in HTML. +//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the +//! HTML spec][html] for more info. +//! +//! ## Recommendation +//! +//! Always use heading (atx), never heading (setext). +//! //! ## Tokens //! //! * [`HeadingAtx`][Name::HeadingAtx] @@ -51,7 +59,7 @@ //! [heading_setext]: crate::construct::heading_setext //! [hard_break_escape]: crate::construct::hard_break_escape //! [html]: https://html.spec.whatwg.org/multipage/sections.html#the-h1,-h2,-h3,-h4,-h5,-and-h6-elements -//! [wiki-setext]: https://en.wikipedia.org/wiki/Setext +//! [wiki_setext]: https://en.wikipedia.org/wiki/Setext //! [atx]: http://www.aaronsw.com/2002/atx/ use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE}; diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index 043104a..bad781c 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -1,20 +1,21 @@ -//! Heading (setext) is a construct that occurs in the [flow] content type. +//! Heading (setext) occurs in the [flow][] content type. //! -//! They’re formed with the following BNF: +//! ## Grammar +//! +//! Heading (setext) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf -//! heading_setext ::= line *(eol line) eol whitespace_optional (1*'-' | 1*'=') whitespace_optional +//! heading_setext ::= paragraph eol *space_or_tab (1*'-' | 1*'=') *space_or_tab //! -//! whitespace ::= 1*space_or_tab -//! whitespace_optional ::= [ whitespace ] -//! line ::= code - eol -//! eol ::= '\r' | '\r\n' | '\n' +//! ; See the `paragraph` construct for the BNF of that part. //! ``` //! -//! Heading (setext) in markdown relates to the `<h1>` and `<h2>` elements in -//! HTML. -//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the -//! HTML spec][html] for more info. +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). +//! +//! See [`paragraph`][paragraph] for grammar, notes, and recommendations on +//! that part. //! //! In markdown, it is also possible to create headings with a //! [heading (atx)][heading_atx] construct. @@ -23,7 +24,6 @@ //! [hard break (escape)][hard_break_escape]). //! However, their limit is that they cannot form `<h3>` through `<h6>` //! headings. -//! Due to this limitation, it is recommended to use atx headings. //! //! [Thematic breaks][thematic_break] formed with dashes and without whitespace //! could be interpreted as a heading (setext). @@ -32,12 +32,23 @@ //! //! > 🏛 **Background**: the word *setext* originates from a small markup //! > language by Ian Feldman from 1991. -//! > See [*§ Setext* on Wikipedia][wiki-setext] for more info. +//! > See [*§ Setext* on Wikipedia][wiki_setext] for more info. //! > The word *atx* originates from a tiny markup language by Aaron Swartz //! > from 2002. //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for //! > more info. //! +//! ## HTML +//! +//! Heading (setext) in markdown relates to the `<h1>` and `<h2>` elements in +//! HTML. +//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the +//! HTML spec][html] for more info. +//! +//! ## Recommendation +//! +//! Always use heading (atx), never heading (setext). +//! //! ## Tokens //! //! * [`HeadingSetext`][Name::HeadingSetext] @@ -50,11 +61,12 @@ //! * [*§ 4.3 Setext headings* in `CommonMark`](https://spec.commonmark.org/0.30/#setext-headings) //! //! [flow]: crate::construct::flow +//! [paragraph]: crate::construct::paragraph //! [heading_atx]: crate::construct::heading_atx //! [thematic_break]: crate::construct::thematic_break //! [hard_break_escape]: crate::construct::hard_break_escape //! [html]: https://html.spec.whatwg.org/multipage/sections.html#the-h1,-h2,-h3,-h4,-h5,-and-h6-elements -//! [wiki-setext]: https://en.wikipedia.org/wiki/Setext +//! [wiki_setext]: https://en.wikipedia.org/wiki/Setext //! [atx]: http://www.aaronsw.com/2002/atx/ use crate::constant::TAB_SIZE; diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index 38e33f8..bd41aa9 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -1,38 +1,38 @@ -//! HTML (flow) is a construct that occurs in the [flow][] cont&ent type. +//! HTML (flow) occurs in the [flow][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! HTML (flow) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! html_flow ::= raw | comment | instruction | declaration | cdata | basic | complete //! -//! ; Note: closing tag name need to match opening tag name. -//! raw ::= '<' raw_tag_name [ [ ( whitespace | '>' ) *line ] *( eol *line ) ] [ '</' raw_tag_name *line ] -//! comment ::= '<!--' [ *'-' '>' *line | *line *( eol *line ) [ '-->' *line ] ] -//! instruction ::= '<?' [ '>' *line | *line *( eol *line ) [ '?>' *line ] ] -//! declaration ::= '<!' ascii_alphabetic *line *( eol *line ) [ '>' *line ] -//! cdata ::= '<![CDATA[' *line *( eol *line ) [ ']]>' *line ] -//! basic ::= '< [ '/' ] basic_tag_name [ [ '/' ] '>' *line *( eol 1*line ) ] -//! complete ::= ( opening_tag | closing_tag ) ( whitespace_optional *( eol 1*line ) | whitespace_optional ) +//! ; Note: closing tag name does not need to match opening tag name. +//! raw ::= '<' raw_tag_name [[space_or_tab *line | '>' *line] eol] *(*line eol) ['</' raw_tag_name *line] +//! comment ::= '<!--' [*'-' '>' *line | *line *(eol *line) ['-->' *line]] +//! instruction ::= '<?' ['>' *line | *line *(eol *line) ['?>' *line]] +//! declaration ::= '<!' ascii_alphabetic *line *(eol *line) ['>' *line] +//! cdata ::= '<![CDATA[' *line *(eol *line) [']]>' *line] +//! basic ::= '< ['/'] basic_tag_name [['/'] '>' *line *(eol 1*line)] +//! complete ::= (opening_tag | closing_tag) [*space_or_tab *(eol 1*line)] //! //! raw_tag_name ::= 'pre' | 'script' | 'style' | 'textarea' ; Note: case-insensitive. //! basic_tag_name ::= 'address' | 'article' | 'aside' | ... ; See `constants.rs`, and note: case-insensitive. -//! opening_tag ::= '<' tag_name *( whitespace attribute ) [ whitespace_optional '/' ] whitespace_optional '>' -//! closing_tag ::= '</' tag_name whitespace_optional '>' -//! tag_name ::= ascii_alphabetic *( '-' | ascii_alphanumeric ) -//! attribute ::= attribute_name [ whitespace_optional '=' whitespace_optional attribute_value ] -//! attribute_name ::= ( ':' | '_' | ascii_alphabetic ) *( '-' | '.' | ':' | '_' | ascii_alphanumeric ) -//! attribute_value ::= '"' *( line - '"' ) '"' | "'" *( line - "'" ) "'" | 1*( line - space_or_tab - '"' - "'" - '/' - '<' - '=' - '>' - '`') -//! -//! whitespace ::= 1*space_or_tab -//! whitespace_optional ::= [ whitespace ] -//! line ::= code - eol -//! eol ::= '\r' | '\r\n' | '\n' -//! space_or_tab ::= ' ' | '\t' +//! opening_tag ::= '<' tag_name *(space_or_tab_eol attribute) [[space_or_tab_eol] '/'] [space_or_tab_eol] '>' +//! closing_tag ::= '</' tag_name [space_or_tab_eol] '>' +//! tag_name ::= ascii_alphabetic *('-' | ascii_alphanumeric) +//! attribute ::= attribute_name [[space_or_tab_eol] '=' [space_or_tab_eol] attribute_value] +//! attribute_name ::= (':' | '_' | ascii_alphabetic) *('-' | '.' | ':' | '_' | ascii_alphanumeric) +//! attribute_value ::= '"' *(line - '"') '"' | "'" *(line - "'") "'" | 1*(text - '"' - "'" - '/' - '<' - '=' - '>' - '`') //! ``` //! +//! As this construct occurs in flow, like all flow constructs, it must be +//! followed by an eol (line ending) or eof (end of file). +//! //! The grammar for HTML in markdown does not resemble the rules of parsing //! HTML according to the [*§ 13.2 Parsing HTML documents* in the HTML -//! spec][html-parsing]. +//! spec][html_parsing]. //! As such, HTML in markdown *resembles* HTML, but is instead a (naïve?) //! attempt to parse an XML-like language. //! By extension, another notable property of the grammar is that it can @@ -96,7 +96,7 @@ //! [paragraph]: crate::construct::paragraph //! [html_raw_names]: crate::constant::HTML_RAW_NAMES //! [html_block_names]: crate::constant::HTML_BLOCK_NAMES -//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing +//! [html_parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing use crate::constant::{ HTML_BLOCK_NAMES, HTML_CDATA_PREFIX, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_SIZE, diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index fde0847..26eded9 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -1,34 +1,31 @@ -//! HTML (text) is a construct that occurs in the [text][] content type. +//! HTML (text) occurs in the [text][] content type. //! -//! It forms with the following BNF: +//! ## Grammar +//! +//! HTML (text) forms with the following BNF +//! (<small>see [construct][crate::construct] for character groups</small>): //! //! ```bnf //! html_text ::= comment | instruction | declaration | cdata | tag_close | tag_open //! //! ; Restriction: the text is not allowed to start with `>`, `->`, or to contain `--`. -//! comment ::= '<!--' *code '-->' -//! instruction ::= '<?' *code '?>' -//! declaration ::= '<!' ascii_alphabetic *code '>' +//! comment ::= '<!--' *byte '-->' +//! instruction ::= '<?' *byte '?>' +//! declaration ::= '<!' ascii_alphabetic *byte '>' //! ; Restriction: the text is not allowed to contain `]]`. -//! cdata ::= '<![CDATA[' *code ']]>' -//! tag_close ::= '</' tag_name whitespace_optional '>' -//! opening_tag ::= '<' tag_name *( whitespace attribute ) [ whitespace_optional '/' ] whitespace_optional '>' +//! cdata ::= '<![CDATA[' *byte ']]>' +//! tag_close ::= '</' tag_name [space_or_tab_eol] '>' +//! opening_tag ::= '<' tag_name *(space_or_tab_eol attribute) [[space_or_tab_eol] '/'] [space_or_tab_eol] '>' //! //! tag_name ::= ascii_alphabetic *( '-' | ascii_alphanumeric ) -//! attribute ::= attribute_name [ whitespace_optional '=' whitespace_optional attribute_value ] -//! attribute_name ::= ( ':' | '_' | ascii_alphabetic ) *( '-' | '.' | ':' | '_' | ascii_alphanumeric ) -//! attribute_value ::= '"' *( code - '"' ) '"' | "'" *( code - "'" ) "'" | 1*( code - space_or_tab - eol - '"' - "'" - '/' - '<' - '=' - '>' - '`') -//! -//! ; Note: blank lines can never occur in `text`. -//! whitespace ::= 1*space_or_tab | [ *space_or_tab eol *space_or_tab ] -//! whitespace_optional ::= [ whitespace ] -//! eol ::= '\r' | '\r\n' | '\n' -//! space_or_tab ::= ' ' | '\t' +//! attribute ::= attribute_name [[space_or_tab_eol] '=' [space_or_tab_eol] attribute_value] +//! attribute_name ::= (':' | '_' | ascii_alphabetic) *('-' | '.' | ':' | '_' | ascii_alphanumeric) +//! attribute_value ::= '"' *(byte - '"') '"' | "'" *(byte - "'") "'" | 1*(text - '"' - "'" - '/' - '<' - '=' - '>' - '`') //! ``` //! //! The grammar for HTML in markdown does not resemble the rules of parsing //! HTML according to the [*§ 13.2 Parsing HTML documents* in the HTML -//! spec][html-parsing]. +//! spec][html_parsing]. //! See the related flow construct [HTML (flow)][html_flow] for more info. //! //! Because the **tag open** and **tag close** productions in the grammar form @@ -52,7 +49,7 @@ //! //! [text]: crate::construct::text //! [html_flow]: crate::construct::html_flow -//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing +//! [html_parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing use crate::constant::HTML_CDATA_PREFIX; use crate::construct::partial_space_or_tab::space_or_tab; diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index 09716b7..4752639 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -1,4 +1,4 @@ -//! Label end is a construct that occurs in the [text][] conten&t type. +//! Label end is a construct that occurs in the [text][] content type. //! //! It forms with the following BNF: //! diff --git a/src/construct/list_item.rs b/src/construct/list_item.rs index a70906a..09678dd 100644 --- a/src/construct/list_item.rs +++ b/src/construct/list_item.rs @@ -289,7 +289,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { container.size = prefix; tokenizer.exit(Name::ListItemPrefix); - tokenizer.register_resolver_before(ResolveName::List); + tokenizer.register_resolver_before(ResolveName::ListItem); State::Ok } } diff --git a/src/construct/mod.rs b/src/construct/mod.rs index 49868e9..da2f5e8 100644 --- a/src/construct/mod.rs +++ b/src/construct/mod.rs @@ -1,17 +1,33 @@ //! Constructs found in markdown. //! -//! There are several *things* found when parsing markdown, such as, say, a -//! thematic break. -//! These things are called constructs here. -//! Sometimes, there are several constructs that result in an equivalent thing. -//! For example, [code (fenced)][code_fenced] and -//! [code (indented)][code_indented] are considered different constructs +//! Constructs are grouped by content type. +//! Which content type is allowed somewhere, defines which constructs are +//! allowed there. +//! +//! ## Content type +//! +//! The following content types are found in markdown: +//! +//! * [document][] +//! * [flow][] +//! * [string][] +//! * [text][] //! //! Content types also have a *rest* thing: after all things are parsed, //! there’s something left. +//! In document, that is [flow][]. //! In flow, that is a [paragraph][]. //! In string and text, that is [data][partial_data]. //! +//! ## Construct +//! +//! There are several *things* found when parsing markdown, such as, say, a +//! thematic break. +//! These things are called constructs here. +//! Sometimes, there are several constructs that result in an equivalent thing. +//! For example, [code (fenced)][code_fenced] and +//! [code (indented)][code_indented] are considered different constructs. +//! //! The following constructs are found in markdown: //! //! * [attention (strong, emphasis)][attention] @@ -39,7 +55,7 @@ //! > 👉 **Note**: for performance reasons, hard break (trailing) is formed by //! > [whitespace][partial_whitespace]. //! -//! There are also several routines used in different places: +//! There are also several small subroutines typically used in different places: //! //! * [bom][partial_bom] //! * [data][partial_data] @@ -51,20 +67,60 @@ //! * [title][partial_title] //! * [whitespace][partial_whitespace] //! +//! ## Grammar +//! //! Each construct maintained here is explained with a BNF diagram. +//! +//! Such diagrams are considered to be *non-normative*. +//! That is to say, they form illustrative, imperfect, but useful, examples. +//! The code, in Rust, is considered to be normative. +//! //! For example, the docs for [character escape][character_escape] contain: //! //! ```bnf //! character_escape ::= '\\' ascii_punctuation //! ``` //! -//! Such diagrams are considered to be *non-normative*. -//! That is to say, they form illustrative, imperfect, but useful, examples. -//! The code, in Rust, is considered to be normative. +//! These diagrams contain references to character group as defined by Rust on +//! for example [char][], but also often on [u8][], which is what `micromark-rs` +//! typically works on. +//! So, for example, `ascii_punctuation` refers to +//! [`u8::is_ascii_punctuation`][u8::is_ascii_punctuation]. //! -//! They also contain references to character as defined by [char][], so for -//! example `ascii_punctuation` refers to -//! [`char::is_ascii_punctuation`][char::is_ascii_punctuation]. +//! For clarity, the productions used throughout are: +//! +//! ```bnf +//! ; Rust / ASCII groups: +//! ; 'a'..='z' +//! ascii_lowercase ::= 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' +//! ; 'A'..='Z' +//! ascii_uppercase ::= 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' +//! ; 'A'..='Z', 'a'..='z' +//! ascii_alphabetic ::= ascii_lowercase | ascii_uppercase +//! ; '0'..='9' +//! ascii_digit ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' +//! ; '0'..='9'; 'A'..='F', 'a'..='f' +//! ascii_hexdigit ::= ascii_digit | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' +//! ; '0'..='9'; 'A'..='Z', 'a'..='z' +//! ascii_alphanumeric ::= ascii_digit | ascii_alphabetic +//! ; '!'..='/'; ':'..='@'; '['..='`'; '{'..='~' +//! ascii_punctuation ::= '!' | '"' | '#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~' +//! ; 0x00..=0x1F; 0x7F +//! ascii_control ::= 0x00 | 0x01 | 0x02 | 0x03 | 0x04 | 0x05 | 0x06 | 0x07 | 0x08 | 0x09 | 0x0A | 0x0B | 0x0C | 0x0D | 0x0E | 0x0F | 0x10 | 0x12 | 0x13 | 0x14 | 0x15 | 0x16 | 0x17 | 0x18 | 0x19 | 0x1A | 0x1B | 0x1C | 0x1D | 0x1E | 0x1F | 0x7F +//! +//! ; Markdown groups: +//! ; Any byte (u8) +//! byte ::= 0x00..=0xFFFF +//! space_or_tab ::= '\t' | ' ' +//! eol ::= '\n' | '\r' | '\r\n' +//! line ::= byte - eol +//! text ::= line - space_or_tab +//! space_or_tab_eol ::= 1*space_or_tab | 0*space_or_tab eol 0*space_or_tab +//! +//! ; Unicode groups: +//! unicode_whitespace ::= ? ; See `char::is_whitespace`. +//! unicode_punctuation ::= ? ; See `src/unicode.rs`. +//! ``` pub mod attention; pub mod autolink; diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs index 3ffa646..adbfae1 100644 --- a/src/construct/partial_data.rs +++ b/src/construct/partial_data.rs @@ -1,6 +1,6 @@ //! Data occurs in [text][] and [string][]. //! -//! It can include anything (including line endings), and stops at certain +//! It can include anything (except for line endings) and stops at certain //! characters. //! //! [string]: crate::construct::string diff --git a/src/construct/partial_whitespace.rs b/src/construct/partial_whitespace.rs index bf06df9..04016cb 100644 --- a/src/construct/partial_whitespace.rs +++ b/src/construct/partial_whitespace.rs @@ -71,7 +71,7 @@ pub fn resolve_whitespace(tokenizer: &mut Tokenizer, hard_break: bool, trim_whol } } -/// Trim a [`Data`][Name::Data] token. +/// Trim a [`Data`][Name::Data] event. fn trim_data( tokenizer: &mut Tokenizer, exit_index: usize, @@ -109,7 +109,7 @@ fn trim_data( }; // The whole data is whitespace. - // We can be very fast: we only change the token types. + // We can be very fast: we only change the event names. if index == 0 { tokenizer.events[exit_index - 1].name = name.clone(); tokenizer.events[exit_index].name = name; @@ -157,7 +157,7 @@ fn trim_data( } // The whole data is whitespace. - // We can be very fast: we only change the token types. + // We can be very fast: we only change the event names. if index == slice.bytes.len() { tokenizer.events[exit_index - 1].name = Name::SpaceOrTab; tokenizer.events[exit_index].name = Name::SpaceOrTab; |