aboutsummaryrefslogtreecommitdiffstats
path: root/src/construct
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/construct/attention.rs31
-rw-r--r--src/construct/autolink.rs52
-rw-r--r--src/construct/blank_line.rs35
-rw-r--r--src/construct/block_quote.rs23
-rw-r--r--src/construct/character_escape.rs19
-rw-r--r--src/construct/character_reference.rs27
-rw-r--r--src/construct/code_fenced.rs73
-rw-r--r--src/construct/code_indented.rs44
-rw-r--r--src/construct/code_text.rs32
-rw-r--r--src/construct/definition.rs42
-rw-r--r--src/construct/document.rs4
-rw-r--r--src/construct/hard_break_escape.rs29
-rw-r--r--src/construct/heading_atx.rs32
-rw-r--r--src/construct/heading_setext.rs40
-rw-r--r--src/construct/html_flow.rs48
-rw-r--r--src/construct/html_text.rs35
-rw-r--r--src/construct/label_end.rs2
-rw-r--r--src/construct/list_item.rs2
-rw-r--r--src/construct/mod.rs82
-rw-r--r--src/construct/partial_data.rs2
-rw-r--r--src/construct/partial_whitespace.rs6
21 files changed, 419 insertions, 241 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index 1dc8868..21407b7 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -1,22 +1,29 @@
-//! Attention is a construct that occurs in the [text][] content type.
+//! Attention (emphasis and strong) occurs in the [text][] content type.
//!
-//! How attention parses is too complex to explain in BNF.
-//! Essentially, one or more of `*` or `_` form attention sequences.
-//! Depending on the code before and after a sequence, it can open or close
-//! attention.
-//! When everything is parsed, we find each sequence that can close, and a
-//! corresponding sequence that can open which uses the same marker.
-//! If both sequences have two or more markers, strong is formed.
-//! Otherwise emphasis is formed.
+//! ## Grammar
//!
-//! Attention sequences do not, on their own, relate to anything in HTML.
-//! When matched with another sequence, and two markers can be “taken” from
-//! them, they together relate to the `<strong>` element in HTML.
+//! Attention sequences form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
+//!
+//! ```bnf
+//! attention_sequence ::= 1*'*' | 1*'_'
+//! ```
+//!
+//! Sequences are matched together to form attention based on which character
+//! they contain, and what character occurs before and after each sequence.
+//! Otherwise they are turned into data.
+//!
+//! ## HTML
+//!
+//! When sequences match, and two markers can be “taken” from them, they
+//! together relate to the `<strong>` element in HTML.
//! When one marker can be taken, they relate to the `<em>` element.
//! See [*§ 4.5.2 The `em` element*][html-em] and
//! [*§ 4.5.3 The `strong` element*][html-strong] in the HTML spec for more
//! info.
//!
+//! ## Recommendation
+//!
//! It is recommended to use asterisks for attention when writing markdown.
//!
//! There are some small differences in whether sequences can open and/or close
diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index 37e21d9..9890aaf 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -1,22 +1,24 @@
-//! Autolinks are a construct that occurs in the [text][] content type.
+//! Autolinks occur in the [text][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! Autolinks form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! autolink ::= '<' ( url | email ) '>'
+//! autolink ::= '<' (url | email) '>'
+//!
+//! url ::= protocol *url_byte
+//! protocol ::= ascii_alphabetic 0*31(protocol_byte) ':'
+//! protocol_byte ::= '+' '-' '.' ascii_alphanumeric
+//! url_byte ::= byte - ascii_control - ' '
//!
-//! url ::= ascii_alphabetic 0*31( '+' '-' '.' ascii_alphanumeric ) ':' *( code - ascii_control - '\r' - '\n' - ' ')
-//! email ::= 1*ascii_atext '@' domain *('.' domain)
+//! email ::= 1*ascii_atext '@' email_domain *('.' email_domain)
//! ; Restriction: up to (including) 63 character are allowed in each domain.
-//! domain ::= ascii_alphanumeric *( ascii_alphanumeric | '-' ascii_alphanumeric )
-//! ascii_atext ::= ascii_alphanumeric | '#' .. '\'' | '*' | '+' | '-' | '/' | '=' | '?' | '^' .. '`' | '{' .. '~'
-//! ```
+//! email_domain ::= ascii_alphanumeric *(ascii_alphanumeric | '-' ascii_alphanumeric)
//!
-//! Autolinks relate to the `<a>` element in HTML.
-//! See [*§ 4.5.1 The `a` element*][html-a] in the HTML spec for more info.
-//! When an email autolink is used (so, without a protocol), the string
-//! `mailto:` is prepended before the email, when generating the `href`
-//! attribute of the hyperlink.
+//! ascii_atext ::= ascii_alphanumeric | '!' | '"' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | '-' | '/' | '=' | '?' | '^' | '_' | '`' | '{' | '|' | '}' | '~'
+//! ```
//!
//! The maximum allowed size of a scheme is `31` (inclusive), which is defined
//! in [`AUTOLINK_SCHEME_SIZE_MAX`][autolink_scheme_size_max].
@@ -41,7 +43,7 @@
//! There are several cases where incorrect encoding of URLs would, in other
//! languages, result in a parse error.
//! In markdown, there are no errors, and URLs are normalized.
-//! In addition, unicode characters are percent encoded
+//! In addition, many characters are percent encoded
//! ([`sanitize_uri`][sanitize_uri]).
//! For example:
//!
@@ -82,6 +84,22 @@
//! <p><a href="#"></a><a href="https://example.com">https://example.com</a></p>
//! ```
//!
+//! ## HTML
+//!
+//! Autolinks relate to the `<a>` element in HTML.
+//! See [*§ 4.5.1 The `a` element*][html_a] in the HTML spec for more info.
+//! When an email autolink is used (so, without a protocol), the string
+//! `mailto:` is prepended before the email, when generating the `href`
+//! attribute of the hyperlink.
+//!
+//! ## Recommendation
+//!
+//! It is recommended to use labels ([label start link][label_start_link],
+//! [label end][label_end]), either with a resource or a definition
+//! ([definition][]), instead of autolinks, as those allow more characters in
+//! URLs, and allow relative URLs and `www.` URLs.
+//! They also allow for descriptive text to explain the URL in prose.
+//!
//! ## Tokens
//!
//! * [`Autolink`][Name::Autolink]
@@ -95,11 +113,13 @@
//! * [*§ 6.4 Autolinks* in `CommonMark`](https://spec.commonmark.org/0.30/#autolinks)
//!
//! [text]: crate::construct::text
+//! [definition]: crate::construct::definition
+//! [label_start_link]: crate::construct::label_start_link
//! [label_end]: crate::construct::label_end
//! [autolink_scheme_size_max]: crate::constant::AUTOLINK_SCHEME_SIZE_MAX
//! [autolink_domain_size_max]: crate::constant::AUTOLINK_DOMAIN_SIZE_MAX
//! [sanitize_uri]: crate::util::sanitize_uri
-//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
+//! [html_a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX};
use crate::event::Name;
@@ -293,7 +313,7 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.size = 0;
let index = tokenizer.events.len();
tokenizer.exit(Name::AutolinkProtocol);
- // Change the token type.
+ // Change the event name.
tokenizer.events[index - 1].name = Name::AutolinkEmail;
tokenizer.events[index].name = Name::AutolinkEmail;
tokenizer.enter(Name::AutolinkMarker);
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index 928b8cc..5be406d 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -1,22 +1,39 @@
-//! Blank lines are a construct that occurs in the [flow][] content type.
+//! Blank lines occur in the [flow][] content type.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Blank lines form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! blank_line ::= *(' ' '\t')
+//! blank_line ::= *space_or_tab
//! ```
//!
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
+//!
//! Blank lines are sometimes needed, such as to differentiate a [paragraph][]
//! from another paragraph.
//! In several cases, blank lines are not needed between flow constructs,
-//! such as between two [heading (atx)][heading-atx]s.
+//! such as between two [heading (atx)][heading_atx]s.
//! Sometimes, whether blank lines are present, changes the behavior of how
-//! HTML is rendered, such as whether blank lines are present between list
-//! items in a [list][list-item].
+//! HTML is rendered, such as whether blank lines are present inside or between
+//! [list items][list_item].
//! More than one blank line is never needed in `CommonMark`.
//!
//! Because blank lines can be empty (line endings are not considered part of
-//! it), and events cannot be empty, blank lines are not present as a token.
+//! it), and events cannot be empty, blank lines are not present as an event.
+//!
+//! ## HTML
+//!
+//! Blank lines do not relate an element in HTML, except for the role they play
+//! when inside or between [list items][list_item].
+//!
+//! ## Recommendation
+//!
+//! It is recommended to always use a blank line between every flow construct,
+//! to use blank lines (consistently) between list items as desired, and to
+//! never use more than one blank line.
//!
//! ## Tokens
//!
@@ -27,8 +44,8 @@
//! * [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js)
//! * [*§ 4.9 Blank lines* in `CommonMark`](https://spec.commonmark.org/0.30/#blank-lines)
//!
-//! [heading-atx]: crate::construct::heading_atx
-//! [list-item]: crate::construct::list_item
+//! [heading_atx]: crate::construct::heading_atx
+//! [list_item]: crate::construct::list_item
//! [paragraph]: crate::construct::paragraph
//! [flow]: crate::construct::flow
diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs
index 37726c5..8d7e227 100644
--- a/src/construct/block_quote.rs
+++ b/src/construct/block_quote.rs
@@ -1,6 +1,9 @@
-//! Block quote is a construct that occurs in the [document][] content type.
+//! Block quotes occur in the [document][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! Block quotes form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! block_quote_start ::= '>' [ space_or_tab ]
@@ -9,14 +12,24 @@
//!
//! Further lines that are not prefixed with `block_quote_cont` cause the block
//! quote to be exited, except when those lines are lazy continuation.
-//! Like so many things in markdown, block quotes too, are very complex.
-//! See [*§ Phase 1: block structure*][commonmark-block] for more on parsing
-//! details.
+//! Like so many things in markdown, block quotes too, are complex.
+//! See [*§ Phase 1: block structure* in `CommonMark`][commonmark-block] for
+//! more on parsing details.
+//!
+//! As block quote is a container, it takes several bytes from the start of the
+//! line, while the rest of the line includes more containers or flow.
+//!
+//! ## HTML
//!
//! Block quote relates to the `<blockquote>` element in HTML.
//! See [*§ 4.4.4 The `blockquote` element*][html-blockquote] in the HTML spec
//! for more info.
//!
+//! ## Recommendation
+//!
+//! Always use a single space after a block quote marker (`>`).
+//! Never use lazy continuation.
+//!
//! ## Tokens
//!
//! * [`BlockQuote`][Name::BlockQuote]
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index 6dac458..438092e 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -1,7 +1,9 @@
-//! Character escapes are a construct that occurs in the [string][] and
-//! [text][] content types.
+//! Character escapes occur in the [string][] and [text][] content types.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Character escapes form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! character_escape ::= '\\' ascii_punctuation
@@ -10,13 +12,20 @@
//! Like much of markdown, there are no “invalid” character escapes: just a
//! slash, or a slash followed by anything other than an ASCII punctuation
//! character, is exactly that: just a slash.
-//! To escape (most) arbitrary characters, use a
-//! [character reference][character_reference] instead
+//!
+//! To escape (almost all) arbitrary characters instead of only ASCII
+//! punctuation, use a [character reference][character_reference] instead
//! (as in, `&amp;`, `&#123;`, or say `&#x9;`).
+//!
//! It is also possible to escape a line ending in text with a similar
//! construct: a [hard break (escape)][hard_break_escape] is a backslash followed
//! by a line ending (that is part of the construct instead of ending it).
//!
+//! ## Recommendation
+//!
+//! If possible, use a character escape.
+//! Otherwise, use a character reference.
+//!
//! ## Tokens
//!
//! * [`CharacterEscape`][Name::CharacterEscape]
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index 7935109..3bdc636 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -1,25 +1,27 @@
-//! Character references are a construct that occurs in the [string][] and
-//! [text][] content types.
+//! Character references occur in the [string][] and [text][] content types.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Character references form with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! character_reference ::= '&' (numeric | named) ';'
//!
//! numeric ::= '#' (hexadecimal | decimal)
-//! ; Note: Limit of `6` imposed as all bigger numbers are invalid:
+//! ; Note: Limit of `6` imposed, as all bigger numbers are invalid.
//! hexadecimal ::= ('x' | 'X') 1*6(ascii_hexdigit)
-//! ; Note: Limit of `7` imposed as all bigger numbers are invalid:
+//! ; Note: Limit of `7` imposed, as all bigger numbers are invalid.
//! decimal ::= 1*7(ascii_digit)
-//! ; Note: Limit of `31` imposed by `CounterClockwiseContourIntegral`:
+//! ; Note: Limit of `31` imposed, for `CounterClockwiseContourIntegral`.
//! ; Note: Limited to any known named character reference (see `constants.rs`)
//! named ::= 1*31(ascii_alphanumeric)
//! ```
//!
//! Like much of markdown, there are no “invalid” character references.
//! However, for security reasons, several numeric character references parse
-//! fine but are not rendered as their corresponding character and they are
-//! instead replaced by a U+FFFD REPLACEMENT CHARACTER (`�`).
+//! fine but are not rendered as their corresponding character.
+//! They are instead replaced by a U+FFFD REPLACEMENT CHARACTER (`�`).
//! See [`decode_numeric`][decode_numeric] for more info.
//!
//! To escape ASCII punctuation characters, use the terser
@@ -33,13 +35,18 @@
//!
//! Character references are parsed insensitive to casing.
//! The casing of hexadecimal numeric character references has no effect.
-//! The casing of named character references does not matter when parsing them,
-//! but does affect whether they match.
+//! The casing of named character references does not matter when parsing, but
+//! does affect whether they match.
//! Depending on the name, one or more cases are allowed, such as that `AMP`
//! and `amp` are both allowed but other cases are not.
//! See [`CHARACTER_REFERENCES`][character_references] for which
//! names match.
//!
+//! ## Recommendation
+//!
+//! If possible, use a character escape.
+//! Otherwise, use a character reference.
+//!
//! ## Tokens
//!
//! * [`CharacterReference`][Name::CharacterReference]
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index 3812d44..748e38f 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -1,9 +1,12 @@
-//! Code (fenced) is a construct that occurs in the [flow][] content type.
+//! Code (fenced) occurs in the [flow][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! Code (fenced) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! code_fenced ::= fence_open *( eol *code ) [ eol fence_close ]
+//! code_fenced ::= fence_open *( eol *byte ) [ eol fence_close ]
//!
//! fence_open ::= sequence [ 1*space_or_tab info [ 1*space_or_tab meta ] ] *space_or_tab
//! ; Restriction: the number of markers in the closing fence sequence must be
@@ -13,41 +16,53 @@
//! ; marker in the opening fence sequence
//! fence_close ::= sequence *space_or_tab
//! sequence ::= 3*'`' | 3*'~'
+//! ; Restriction: the `` ` `` character cannot occur in `info` if it is the marker.
//! info ::= 1*text
+//! ; Restriction: the `` ` `` character cannot occur in `meta` if it is the marker.
//! meta ::= 1*text *( *space_or_tab 1*text )
-//!
-//! ; Restriction: the `` ` `` character cannot occur in `text` if it is the
-//! ; marker of the opening fence sequence.
-//! text ::= code - eol - space_or_tab
-//! eol ::= '\r' | '\r\n' | '\n'
-//! space_or_tab ::= ' ' | '\t'
-//! code ::= . ; any unicode code point (other than line endings).
//! ```
//!
-//! The above grammar does not show how whitespace is handled.
-//! To parse code (fenced), let `X` be the number of whitespace characters
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
+//!
+//! The above grammar does not show how indentation (with `space_or_tab`) of
+//! each line is handled.
+//! To parse code (fenced), let `x` be the number of `space_or_tab` characters
//! before the opening fence sequence.
//! Each line of text is then allowed (not required) to be indented with up
-//! to `X` spaces or tabs, which are then ignored as an indent instead of being
+//! to `x` spaces or tabs, which are then ignored as an indent instead of being
//! considered as part of the code.
//! This indent does not affect the closing fence.
//! It can be indented up to a separate 3 spaces or tabs.
//! A bigger indent makes it part of the code instead of a fence.
//!
-//! Code (fenced) relates to both the `<pre>` and the `<code>` elements in
-//! HTML.
-//! See [*§ 4.4.3 The `pre` element*][html-pre] and the [*§ 4.5.15 The `code`
-//! element*][html-code] in the HTML spec for more info.
+//! The `info` and `meta` parts are interpreted as the [string][] content type.
+//! That means that [character escapes][character_escape] and
+//! [character references][character_reference] are allowed.
//!
//! The optional `meta` part is ignored: it is not used when parsing or
//! rendering.
+//!
//! The optional `info` part is used and is expected to specify the programming
//! language that the code is in.
//! Which value it holds depends on what your syntax highlighter supports, if
//! one is used.
+//!
+//! In markdown, it is also possible to use [code (text)][code_text] in the
+//! [text][] content type.
+//! It is also possible to create code with the
+//! [code (indented)][code_indented] construct.
+//!
+//! ## HTML
+//!
+//! Code (fenced) relates to both the `<pre>` and the `<code>` elements in
+//! HTML.
+//! See [*§ 4.4.3 The `pre` element*][html_pre] and the [*§ 4.5.15 The `code`
+//! element*][html_code] in the HTML spec for more info.
+//!
//! The `info` is, when rendering to HTML, typically exposed as a class.
//! This behavior stems from the HTML spec ([*§ 4.5.15 The `code`
-//! element*][html-code]).
+//! element*][html_code]).
//! For example:
//!
//! ```markdown
@@ -63,17 +78,11 @@
//! </code></pre>
//! ```
//!
-//! The `info` and `meta` parts are interpreted as the [string][] content type.
-//! That means that [character escapes][character_escape] and
-//! [character references][character_reference] are allowed.
+//! ## Recommendation
//!
-//! In markdown, it is also possible to use [code (text)][code_text] in the
-//! [text][] content type.
-//! It is also possible to create code with the
-//! [code (indented)][code_indented] construct.
-//! That construct is less explicit, different from code (text), and has no
-//! support for specifying the programming language, so it is recommended to
-//! use code (fenced) instead of code (indented).
+//! It is recommended to use code (fenced) instead of code (indented).
+//! Code (fenced) is more explicit, similar to code (text), and has support
+//! for specifying the programming language.
//!
//! ## Tokens
//!
@@ -94,12 +103,12 @@
//! [flow]: crate::construct::flow
//! [string]: crate::construct::string
//! [text]: crate::construct::text
-//! [code_indented]: crate::construct::code_indented
-//! [code_text]: crate::construct::code_text
//! [character_escape]: crate::construct::character_escape
//! [character_reference]: crate::construct::character_reference
-//! [html-pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element
-//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
+//! [code_indented]: crate::construct::code_indented
+//! [code_text]: crate::construct::code_text
+//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
+//! [html_pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element
use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE};
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index e3a5333..89c5652 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -1,30 +1,38 @@
-//! Code (indented) is a construct that occurs in the [flow][] content type.
+//! Code (indented) occurs in the [flow][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! Code (indented) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! code_indented ::= indented_filled_line *( eol *( blank_line eol ) indented_filled_line )
+//! code_indented ::= filled_line *( eol *( blank_line eol ) filled_line )
//!
-//! ; Restriction: at least one `code` must not be whitespace.
-//! indented_filled_line ::= 4space_or_tab *code
+//! ; Restriction: at least one `line` byte must be `text`.
+//! filled_line ::= 4(space_or_tab) *line
//! blank_line ::= *space_or_tab
-//! eol ::= '\r' | '\r\n' | '\n'
-//! code ::= . ; any unicode code point (other than line endings).
-//! space_or_tab ::= ' ' | '\t'
//! ```
//!
-//! Code (indented) relates to both the `<pre>` and the `<code>` elements in
-//! HTML.
-//! See [*§ 4.4.3 The `pre` element*][html-pre] and the [*§ 4.5.15 The `code`
-//! element*][html-code] in the HTML spec for more info.
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
//!
//! In markdown, it is also possible to use [code (text)][code_text] in the
//! [text][] content type.
//! It is also possible to create code with the [code (fenced)][code_fenced]
//! construct.
-//! That construct is more explicit, more similar to code (text), and has
-//! support for specifying the programming language that the code is in, so it
-//! is recommended to use that instead of indented code.
+//!
+//! ## HTML
+//!
+//! Code (indented) relates to both the `<pre>` and the `<code>` elements in
+//! HTML.
+//! See [*§ 4.4.3 The `pre` element*][html_pre] and the [*§ 4.5.15 The `code`
+//! element*][html_code] in the HTML spec for more info.
+//!
+//! ## Recommendation
+//!
+//! It is recommended to use code (fenced) instead of code (indented).
+//! Code (fenced) is more explicit, similar to code (text), and has support
+//! for specifying the programming language.
//!
//! ## Tokens
//!
@@ -40,10 +48,10 @@
//!
//! [flow]: crate::construct::flow
//! [text]: crate::construct::text
-//! [code_text]: crate::construct::code_text
//! [code_fenced]: crate::construct::code_fenced
-//! [html-pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element
-//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
+//! [code_text]: crate::construct::code_text
+//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
+//! [html_pre]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-pre-element
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::TAB_SIZE;
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index 7ebee96..413b5ee 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -1,12 +1,16 @@
-//! Code (text) is a construct that occurs in the [text][] content type.
+//! Code (text) occurs in the [text][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! Code (text) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! ; Restriction: the number of markers in the closing sequence must be equal
//! ; to the number of markers in the opening sequence.
-//! code_text ::= sequence 1*code sequence
+//! code_text ::= sequence 1*byte sequence
//!
+//! ; Restriction: not preceded or followed by `` ` ``.
//! sequence ::= 1*'`'
//! ```
//!
@@ -18,15 +22,13 @@
//! Include more: `a``b` or include less: ``a`b``.
//! ```
//!
-//! When turning markdown into HTML, each line ending is turned into a space.
-//!
//! It is also possible to include just one grave accent (tick):
//!
//! ```markdown
//! Include just one: `` ` ``.
//! ```
//!
-//! Sequences are “gready”, in that they cannot be preceded or succeeded by
+//! Sequences are “gready”, in that they cannot be preceded or followed by
//! more grave accents (ticks).
//! To illustrate:
//!
@@ -53,17 +55,17 @@
//! if both exist and there is also a non-space in the code, are removed.
//! Line endings, at that stage, are considered as spaces.
//!
-//! Code (text) relates to the `<code>` element in HTML.
-//! See [*§ 4.5.15 The `code` element*][html-code] in the HTML spec for more
-//! info.
-//!
//! In markdown, it is possible to create code with the
//! [code (fenced)][code_fenced] or [code (indented)][code_indented] constructs
//! in the [flow][] content type.
-//! Compared to code (indented), fenced code is more explicit and more similar
-//! to code (text), and it has support for specifying the programming language
-//! that the code is in, so it is recommended to use that instead of indented
-//! code.
+//!
+//! ## HTML
+//!
+//! Code (text) relates to the `<code>` element in HTML.
+//! See [*§ 4.5.15 The `code` element*][html_code] in the HTML spec for more
+//! info.
+//!
+//! When turning markdown into HTML, each line ending is turned into a space.
//!
//! ## Tokens
//!
@@ -81,7 +83,7 @@
//! [text]: crate::construct::text
//! [code_indented]: crate::construct::code_indented
//! [code_fenced]: crate::construct::code_fenced
-//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
+//! [html_code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
use crate::event::Name;
use crate::state::{Name as StateName, State};
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index 8f274ee..071e595 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -1,31 +1,29 @@
-//! Definition is a construct that occurs in the [flow] content type.
+//! Definition occurs in the [flow] content type.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Definition forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! definition ::= label ':' [ whitespace ] destination [ whitespace title ] [ space_or_tab ]
+//! definition ::= label ':' [ space_or_tab_eol ] destination [ space_or_tab_eol title ] [ space_or_tab ]
//!
//! ; See the `destination`, `title`, and `label` constructs for the BNF of
//! ; those parts.
//! ```
//!
-//! See [`destination`][destination], [`label`][label], and [`title`][title]
-//! for grammar, notes, and recommendations.
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
//!
-//! Definitions in markdown do not, on their own, relate to anything in HTML.
-//! When matched with a [label end (reference)][label_end], they together
-//! relate to the `<a>` or `<img>` elements in HTML.
-//! The definition forms its `href` or `src`, and optionally `title`,
-//! attributes.
-//! See [*§ 4.5.1 The `a` element*][html-a] and
-//! [*§ 4.8.3 The `img` element*][html-img] in the HTML spec for more info.
+//! See [`destination`][destination], [`label`][label], and [`title`][title]
+//! for grammar, notes, and recommendations on each part.
//!
//! The `destination`, `label`, and `title` parts are interpreted as the
//! [string][] content type.
//! That means that [character escapes][character_escape] and
//! [character references][character_reference] are allowed.
//!
-//! Definitions match to references through their label.
+//! Definitions match to references through identifiers.
//! To match, both labels must be equal after normalizing with
//! [`normalize_identifier`][normalize_identifier].
//! One definition can match to multiple references.
@@ -57,6 +55,16 @@
//! `<img>` when compiling, see
//! [`sanitize_uri`][sanitize_uri].
//!
+//! ## HTML
+//!
+//! Definitions in markdown do not, on their own, relate to anything in HTML.
+//! When matched with a [label end (reference)][label_end], they together
+//! relate to the `<a>` or `<img>` elements in HTML.
+//! The definition forms its `href` or `src`, and optionally `title`,
+//! attributes.
+//! See [*§ 4.5.1 The `a` element*][html_a] and
+//! [*§ 4.8.3 The `img` element*][html_img] in the HTML spec for more info.
+//!
//! ## Tokens
//!
//! * [`Definition`][Name::Definition]
@@ -84,14 +92,14 @@
//! [string]: crate::construct::string
//! [character_escape]: crate::construct::character_escape
//! [character_reference]: crate::construct::character_reference
-//! [label_end]: crate::construct::label_end
//! [destination]: crate::construct::partial_destination
-//! [title]: crate::construct::partial_title
//! [label]: crate::construct::partial_label
+//! [label_end]: crate::construct::label_end
+//! [title]: crate::construct::partial_title
//! [sanitize_uri]: crate::util::sanitize_uri::sanitize_uri
//! [normalize_identifier]: crate::util::normalize_identifier
-//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
-//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
+//! [html_a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
+//! [html_img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
use crate::construct::partial_space_or_tab::space_or_tab;
use crate::construct::partial_space_or_tab_eol::space_or_tab_eol;
diff --git a/src/construct/document.rs b/src/construct/document.rs
index 9def6c5..0cda368 100644
--- a/src/construct/document.rs
+++ b/src/construct/document.rs
@@ -6,7 +6,7 @@
//! The constructs found in flow are:
//!
//! * [Block quote][crate::construct::block_quote]
-//! * [List][crate::construct::list_item]
+//! * [List item][crate::construct::list_item]
use crate::event::{Content, Event, Kind, Link, Name};
use crate::state::{Name as StateName, State};
@@ -409,7 +409,7 @@ fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) {
}
}
- debug_assert!(found, "expected to find container token to exit");
+ debug_assert!(found, "expected to find container event to exit");
}
if let Some(ref mut list) = tokenizer.tokenize_state.document_exits[index] {
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 1fafa0b..64c909a 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -1,28 +1,33 @@
-//! Hard break (escape) is a construct that occurs in the [text][] content
-//! type.
+//! Hard break (escape) occurs in the [text][] content type.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Hard break (escape) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! ; Restriction: followed by a line ending (that is part of the construct
+//! ; Restriction: followed by a line ending (that is part of the content
//! ; instead of ending it).
//! hard_break_escape ::= '\\'
//! ```
//!
-//! Hard breaks in markdown relate to the HTML element `<br>`.
-//! See [*§ 4.5.27 The `br` element* in the HTML spec][html] for more info.
-//!
//! It is also possible to create a hard break with a
//! [hard break (trailing)][hard_break_trailing].
-//! That construct is not recommended because trailing spaces are typically
-//! invisible in editors, or even automatically removed, making them hard to
-//! use.
//!
-//! It is also possible to escape punctuation characters with a similar
+//! Punctuation characters can be escaped with a similar
//! construct: a [character escape][character_escape] is a backslash followed
//! by an ASCII punctuation character.
//! Arbitrary characters can be escaped with
-//! [character reference][character_reference]s.
+//! [character references][character_reference].
+//!
+//! ## HTML
+//!
+//! Hard breaks in markdown relate to the HTML element `<br>`.
+//! See [*§ 4.5.27 The `br` element* in the HTML spec][html] for more info.
+//!
+//! ## Recommendation
+//!
+//! Always use hard break (escape), never hard break (trailing).
//!
//! ## Tokens
//!
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 3bcff54..960ae32 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -1,17 +1,16 @@
-//! Heading (atx) is a construct that occurs in the [flow] content type.
+//! Heading (atx) occurs in the [flow][] content type.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
//!
-//! ```bnf
-//! heading_atx ::= 1*6'#' [ 1*space_or_tab text [ 1*space_or_tab 1*'#' ] ] *space_or_tab
+//! Heading (atx) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
-//! text ::= code - eol
-//! space_or_tab ::= ' ' | '\t'
+//! ```bnf
+//! heading_atx ::= 1*6'#' [ 1*space_or_tab line [ 1*space_or_tab 1*'#' ] ] *space_or_tab
//! ```
//!
-//! Headings in markdown relate to the `<h1>` through `<h6>` elements in HTML.
-//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the
-//! HTML spec][html] for more info.
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
//!
//! `CommonMark` introduced the requirement on whitespace existing after the
//! opening sequence and before text.
@@ -25,16 +24,25 @@
//! [hard break (escape)][hard_break_escape]).
//! However, their limit is that they cannot form `<h3>` through `<h6>`
//! headings.
-//! Due to this limitation, it is recommended to use atx headings.
//!
//! > 🏛 **Background**: the word *setext* originates from a small markup
//! > language by Ian Feldman from 1991.
-//! > See [*§ Setext* on Wikipedia][wiki-setext] for more info.
+//! > See [*§ Setext* on Wikipedia][wiki_setext] for more info.
//! > The word *atx* originates from a tiny markup language by Aaron Swartz
//! > from 2002.
//! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
//! > more info.
//!
+//! ## HTML
+//!
+//! Headings in markdown relate to the `<h1>` through `<h6>` elements in HTML.
+//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the
+//! HTML spec][html] for more info.
+//!
+//! ## Recommendation
+//!
+//! Always use heading (atx), never heading (setext).
+//!
//! ## Tokens
//!
//! * [`HeadingAtx`][Name::HeadingAtx]
@@ -51,7 +59,7 @@
//! [heading_setext]: crate::construct::heading_setext
//! [hard_break_escape]: crate::construct::hard_break_escape
//! [html]: https://html.spec.whatwg.org/multipage/sections.html#the-h1,-h2,-h3,-h4,-h5,-and-h6-elements
-//! [wiki-setext]: https://en.wikipedia.org/wiki/Setext
+//! [wiki_setext]: https://en.wikipedia.org/wiki/Setext
//! [atx]: http://www.aaronsw.com/2002/atx/
use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE};
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index 043104a..bad781c 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -1,20 +1,21 @@
-//! Heading (setext) is a construct that occurs in the [flow] content type.
+//! Heading (setext) occurs in the [flow][] content type.
//!
-//! They’re formed with the following BNF:
+//! ## Grammar
+//!
+//! Heading (setext) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
-//! heading_setext ::= line *(eol line) eol whitespace_optional (1*'-' | 1*'=') whitespace_optional
+//! heading_setext ::= paragraph eol *space_or_tab (1*'-' | 1*'=') *space_or_tab
//!
-//! whitespace ::= 1*space_or_tab
-//! whitespace_optional ::= [ whitespace ]
-//! line ::= code - eol
-//! eol ::= '\r' | '\r\n' | '\n'
+//! ; See the `paragraph` construct for the BNF of that part.
//! ```
//!
-//! Heading (setext) in markdown relates to the `<h1>` and `<h2>` elements in
-//! HTML.
-//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the
-//! HTML spec][html] for more info.
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
+//!
+//! See [`paragraph`][paragraph] for grammar, notes, and recommendations on
+//! that part.
//!
//! In markdown, it is also possible to create headings with a
//! [heading (atx)][heading_atx] construct.
@@ -23,7 +24,6 @@
//! [hard break (escape)][hard_break_escape]).
//! However, their limit is that they cannot form `<h3>` through `<h6>`
//! headings.
-//! Due to this limitation, it is recommended to use atx headings.
//!
//! [Thematic breaks][thematic_break] formed with dashes and without whitespace
//! could be interpreted as a heading (setext).
@@ -32,12 +32,23 @@
//!
//! > 🏛 **Background**: the word *setext* originates from a small markup
//! > language by Ian Feldman from 1991.
-//! > See [*§ Setext* on Wikipedia][wiki-setext] for more info.
+//! > See [*§ Setext* on Wikipedia][wiki_setext] for more info.
//! > The word *atx* originates from a tiny markup language by Aaron Swartz
//! > from 2002.
//! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
//! > more info.
//!
+//! ## HTML
+//!
+//! Heading (setext) in markdown relates to the `<h1>` and `<h2>` elements in
+//! HTML.
+//! See [*§ 4.3.6 The `h1`, `h2`, `h3`, `h4`, `h5`, and `h6` elements* in the
+//! HTML spec][html] for more info.
+//!
+//! ## Recommendation
+//!
+//! Always use heading (atx), never heading (setext).
+//!
//! ## Tokens
//!
//! * [`HeadingSetext`][Name::HeadingSetext]
@@ -50,11 +61,12 @@
//! * [*§ 4.3 Setext headings* in `CommonMark`](https://spec.commonmark.org/0.30/#setext-headings)
//!
//! [flow]: crate::construct::flow
+//! [paragraph]: crate::construct::paragraph
//! [heading_atx]: crate::construct::heading_atx
//! [thematic_break]: crate::construct::thematic_break
//! [hard_break_escape]: crate::construct::hard_break_escape
//! [html]: https://html.spec.whatwg.org/multipage/sections.html#the-h1,-h2,-h3,-h4,-h5,-and-h6-elements
-//! [wiki-setext]: https://en.wikipedia.org/wiki/Setext
+//! [wiki_setext]: https://en.wikipedia.org/wiki/Setext
//! [atx]: http://www.aaronsw.com/2002/atx/
use crate::constant::TAB_SIZE;
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index 38e33f8..bd41aa9 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -1,38 +1,38 @@
-//! HTML (flow) is a construct that occurs in the [flow][] cont&ent type.
+//! HTML (flow) occurs in the [flow][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! HTML (flow) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! html_flow ::= raw | comment | instruction | declaration | cdata | basic | complete
//!
-//! ; Note: closing tag name need to match opening tag name.
-//! raw ::= '<' raw_tag_name [ [ ( whitespace | '>' ) *line ] *( eol *line ) ] [ '</' raw_tag_name *line ]
-//! comment ::= '<!--' [ *'-' '>' *line | *line *( eol *line ) [ '-->' *line ] ]
-//! instruction ::= '<?' [ '>' *line | *line *( eol *line ) [ '?>' *line ] ]
-//! declaration ::= '<!' ascii_alphabetic *line *( eol *line ) [ '>' *line ]
-//! cdata ::= '<![CDATA[' *line *( eol *line ) [ ']]>' *line ]
-//! basic ::= '< [ '/' ] basic_tag_name [ [ '/' ] '>' *line *( eol 1*line ) ]
-//! complete ::= ( opening_tag | closing_tag ) ( whitespace_optional *( eol 1*line ) | whitespace_optional )
+//! ; Note: closing tag name does not need to match opening tag name.
+//! raw ::= '<' raw_tag_name [[space_or_tab *line | '>' *line] eol] *(*line eol) ['</' raw_tag_name *line]
+//! comment ::= '<!--' [*'-' '>' *line | *line *(eol *line) ['-->' *line]]
+//! instruction ::= '<?' ['>' *line | *line *(eol *line) ['?>' *line]]
+//! declaration ::= '<!' ascii_alphabetic *line *(eol *line) ['>' *line]
+//! cdata ::= '<![CDATA[' *line *(eol *line) [']]>' *line]
+//! basic ::= '< ['/'] basic_tag_name [['/'] '>' *line *(eol 1*line)]
+//! complete ::= (opening_tag | closing_tag) [*space_or_tab *(eol 1*line)]
//!
//! raw_tag_name ::= 'pre' | 'script' | 'style' | 'textarea' ; Note: case-insensitive.
//! basic_tag_name ::= 'address' | 'article' | 'aside' | ... ; See `constants.rs`, and note: case-insensitive.
-//! opening_tag ::= '<' tag_name *( whitespace attribute ) [ whitespace_optional '/' ] whitespace_optional '>'
-//! closing_tag ::= '</' tag_name whitespace_optional '>'
-//! tag_name ::= ascii_alphabetic *( '-' | ascii_alphanumeric )
-//! attribute ::= attribute_name [ whitespace_optional '=' whitespace_optional attribute_value ]
-//! attribute_name ::= ( ':' | '_' | ascii_alphabetic ) *( '-' | '.' | ':' | '_' | ascii_alphanumeric )
-//! attribute_value ::= '"' *( line - '"' ) '"' | "'" *( line - "'" ) "'" | 1*( line - space_or_tab - '"' - "'" - '/' - '<' - '=' - '>' - '`')
-//!
-//! whitespace ::= 1*space_or_tab
-//! whitespace_optional ::= [ whitespace ]
-//! line ::= code - eol
-//! eol ::= '\r' | '\r\n' | '\n'
-//! space_or_tab ::= ' ' | '\t'
+//! opening_tag ::= '<' tag_name *(space_or_tab_eol attribute) [[space_or_tab_eol] '/'] [space_or_tab_eol] '>'
+//! closing_tag ::= '</' tag_name [space_or_tab_eol] '>'
+//! tag_name ::= ascii_alphabetic *('-' | ascii_alphanumeric)
+//! attribute ::= attribute_name [[space_or_tab_eol] '=' [space_or_tab_eol] attribute_value]
+//! attribute_name ::= (':' | '_' | ascii_alphabetic) *('-' | '.' | ':' | '_' | ascii_alphanumeric)
+//! attribute_value ::= '"' *(line - '"') '"' | "'" *(line - "'") "'" | 1*(text - '"' - "'" - '/' - '<' - '=' - '>' - '`')
//! ```
//!
+//! As this construct occurs in flow, like all flow constructs, it must be
+//! followed by an eol (line ending) or eof (end of file).
+//!
//! The grammar for HTML in markdown does not resemble the rules of parsing
//! HTML according to the [*§ 13.2 Parsing HTML documents* in the HTML
-//! spec][html-parsing].
+//! spec][html_parsing].
//! As such, HTML in markdown *resembles* HTML, but is instead a (naïve?)
//! attempt to parse an XML-like language.
//! By extension, another notable property of the grammar is that it can
@@ -96,7 +96,7 @@
//! [paragraph]: crate::construct::paragraph
//! [html_raw_names]: crate::constant::HTML_RAW_NAMES
//! [html_block_names]: crate::constant::HTML_BLOCK_NAMES
-//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
+//! [html_parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
use crate::constant::{
HTML_BLOCK_NAMES, HTML_CDATA_PREFIX, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_SIZE,
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index fde0847..26eded9 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -1,34 +1,31 @@
-//! HTML (text) is a construct that occurs in the [text][] content type.
+//! HTML (text) occurs in the [text][] content type.
//!
-//! It forms with the following BNF:
+//! ## Grammar
+//!
+//! HTML (text) forms with the following BNF
+//! (<small>see [construct][crate::construct] for character groups</small>):
//!
//! ```bnf
//! html_text ::= comment | instruction | declaration | cdata | tag_close | tag_open
//!
//! ; Restriction: the text is not allowed to start with `>`, `->`, or to contain `--`.
-//! comment ::= '<!--' *code '-->'
-//! instruction ::= '<?' *code '?>'
-//! declaration ::= '<!' ascii_alphabetic *code '>'
+//! comment ::= '<!--' *byte '-->'
+//! instruction ::= '<?' *byte '?>'
+//! declaration ::= '<!' ascii_alphabetic *byte '>'
//! ; Restriction: the text is not allowed to contain `]]`.
-//! cdata ::= '<![CDATA[' *code ']]>'
-//! tag_close ::= '</' tag_name whitespace_optional '>'
-//! opening_tag ::= '<' tag_name *( whitespace attribute ) [ whitespace_optional '/' ] whitespace_optional '>'
+//! cdata ::= '<![CDATA[' *byte ']]>'
+//! tag_close ::= '</' tag_name [space_or_tab_eol] '>'
+//! opening_tag ::= '<' tag_name *(space_or_tab_eol attribute) [[space_or_tab_eol] '/'] [space_or_tab_eol] '>'
//!
//! tag_name ::= ascii_alphabetic *( '-' | ascii_alphanumeric )
-//! attribute ::= attribute_name [ whitespace_optional '=' whitespace_optional attribute_value ]
-//! attribute_name ::= ( ':' | '_' | ascii_alphabetic ) *( '-' | '.' | ':' | '_' | ascii_alphanumeric )
-//! attribute_value ::= '"' *( code - '"' ) '"' | "'" *( code - "'" ) "'" | 1*( code - space_or_tab - eol - '"' - "'" - '/' - '<' - '=' - '>' - '`')
-//!
-//! ; Note: blank lines can never occur in `text`.
-//! whitespace ::= 1*space_or_tab | [ *space_or_tab eol *space_or_tab ]
-//! whitespace_optional ::= [ whitespace ]
-//! eol ::= '\r' | '\r\n' | '\n'
-//! space_or_tab ::= ' ' | '\t'
+//! attribute ::= attribute_name [[space_or_tab_eol] '=' [space_or_tab_eol] attribute_value]
+//! attribute_name ::= (':' | '_' | ascii_alphabetic) *('-' | '.' | ':' | '_' | ascii_alphanumeric)
+//! attribute_value ::= '"' *(byte - '"') '"' | "'" *(byte - "'") "'" | 1*(text - '"' - "'" - '/' - '<' - '=' - '>' - '`')
//! ```
//!
//! The grammar for HTML in markdown does not resemble the rules of parsing
//! HTML according to the [*§ 13.2 Parsing HTML documents* in the HTML
-//! spec][html-parsing].
+//! spec][html_parsing].
//! See the related flow construct [HTML (flow)][html_flow] for more info.
//!
//! Because the **tag open** and **tag close** productions in the grammar form
@@ -52,7 +49,7 @@
//!
//! [text]: crate::construct::text
//! [html_flow]: crate::construct::html_flow
-//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
+//! [html_parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
use crate::constant::HTML_CDATA_PREFIX;
use crate::construct::partial_space_or_tab::space_or_tab;
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 09716b7..4752639 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -1,4 +1,4 @@
-//! Label end is a construct that occurs in the [text][] conten&t type.
+//! Label end is a construct that occurs in the [text][] content type.
//!
//! It forms with the following BNF:
//!
diff --git a/src/construct/list_item.rs b/src/construct/list_item.rs
index a70906a..09678dd 100644
--- a/src/construct/list_item.rs
+++ b/src/construct/list_item.rs
@@ -289,7 +289,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
container.size = prefix;
tokenizer.exit(Name::ListItemPrefix);
- tokenizer.register_resolver_before(ResolveName::List);
+ tokenizer.register_resolver_before(ResolveName::ListItem);
State::Ok
}
}
diff --git a/src/construct/mod.rs b/src/construct/mod.rs
index 49868e9..da2f5e8 100644
--- a/src/construct/mod.rs
+++ b/src/construct/mod.rs
@@ -1,17 +1,33 @@
//! Constructs found in markdown.
//!
-//! There are several *things* found when parsing markdown, such as, say, a
-//! thematic break.
-//! These things are called constructs here.
-//! Sometimes, there are several constructs that result in an equivalent thing.
-//! For example, [code (fenced)][code_fenced] and
-//! [code (indented)][code_indented] are considered different constructs
+//! Constructs are grouped by content type.
+//! Which content type is allowed somewhere, defines which constructs are
+//! allowed there.
+//!
+//! ## Content type
+//!
+//! The following content types are found in markdown:
+//!
+//! * [document][]
+//! * [flow][]
+//! * [string][]
+//! * [text][]
//!
//! Content types also have a *rest* thing: after all things are parsed,
//! there’s something left.
+//! In document, that is [flow][].
//! In flow, that is a [paragraph][].
//! In string and text, that is [data][partial_data].
//!
+//! ## Construct
+//!
+//! There are several *things* found when parsing markdown, such as, say, a
+//! thematic break.
+//! These things are called constructs here.
+//! Sometimes, there are several constructs that result in an equivalent thing.
+//! For example, [code (fenced)][code_fenced] and
+//! [code (indented)][code_indented] are considered different constructs.
+//!
//! The following constructs are found in markdown:
//!
//! * [attention (strong, emphasis)][attention]
@@ -39,7 +55,7 @@
//! > 👉 **Note**: for performance reasons, hard break (trailing) is formed by
//! > [whitespace][partial_whitespace].
//!
-//! There are also several routines used in different places:
+//! There are also several small subroutines typically used in different places:
//!
//! * [bom][partial_bom]
//! * [data][partial_data]
@@ -51,20 +67,60 @@
//! * [title][partial_title]
//! * [whitespace][partial_whitespace]
//!
+//! ## Grammar
+//!
//! Each construct maintained here is explained with a BNF diagram.
+//!
+//! Such diagrams are considered to be *non-normative*.
+//! That is to say, they form illustrative, imperfect, but useful, examples.
+//! The code, in Rust, is considered to be normative.
+//!
//! For example, the docs for [character escape][character_escape] contain:
//!
//! ```bnf
//! character_escape ::= '\\' ascii_punctuation
//! ```
//!
-//! Such diagrams are considered to be *non-normative*.
-//! That is to say, they form illustrative, imperfect, but useful, examples.
-//! The code, in Rust, is considered to be normative.
+//! These diagrams contain references to character group as defined by Rust on
+//! for example [char][], but also often on [u8][], which is what `micromark-rs`
+//! typically works on.
+//! So, for example, `ascii_punctuation` refers to
+//! [`u8::is_ascii_punctuation`][u8::is_ascii_punctuation].
//!
-//! They also contain references to character as defined by [char][], so for
-//! example `ascii_punctuation` refers to
-//! [`char::is_ascii_punctuation`][char::is_ascii_punctuation].
+//! For clarity, the productions used throughout are:
+//!
+//! ```bnf
+//! ; Rust / ASCII groups:
+//! ; 'a'..='z'
+//! ascii_lowercase ::= 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z'
+//! ; 'A'..='Z'
+//! ascii_uppercase ::= 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z'
+//! ; 'A'..='Z', 'a'..='z'
+//! ascii_alphabetic ::= ascii_lowercase | ascii_uppercase
+//! ; '0'..='9'
+//! ascii_digit ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
+//! ; '0'..='9'; 'A'..='F', 'a'..='f'
+//! ascii_hexdigit ::= ascii_digit | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F'
+//! ; '0'..='9'; 'A'..='Z', 'a'..='z'
+//! ascii_alphanumeric ::= ascii_digit | ascii_alphabetic
+//! ; '!'..='/'; ':'..='@'; '['..='`'; '{'..='~'
+//! ascii_punctuation ::= '!' | '"' | '#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~'
+//! ; 0x00..=0x1F; 0x7F
+//! ascii_control ::= 0x00 | 0x01 | 0x02 | 0x03 | 0x04 | 0x05 | 0x06 | 0x07 | 0x08 | 0x09 | 0x0A | 0x0B | 0x0C | 0x0D | 0x0E | 0x0F | 0x10 | 0x12 | 0x13 | 0x14 | 0x15 | 0x16 | 0x17 | 0x18 | 0x19 | 0x1A | 0x1B | 0x1C | 0x1D | 0x1E | 0x1F | 0x7F
+//!
+//! ; Markdown groups:
+//! ; Any byte (u8)
+//! byte ::= 0x00..=0xFFFF
+//! space_or_tab ::= '\t' | ' '
+//! eol ::= '\n' | '\r' | '\r\n'
+//! line ::= byte - eol
+//! text ::= line - space_or_tab
+//! space_or_tab_eol ::= 1*space_or_tab | 0*space_or_tab eol 0*space_or_tab
+//!
+//! ; Unicode groups:
+//! unicode_whitespace ::= ? ; See `char::is_whitespace`.
+//! unicode_punctuation ::= ? ; See `src/unicode.rs`.
+//! ```
pub mod attention;
pub mod autolink;
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index 3ffa646..adbfae1 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -1,6 +1,6 @@
//! Data occurs in [text][] and [string][].
//!
-//! It can include anything (including line endings), and stops at certain
+//! It can include anything (except for line endings) and stops at certain
//! characters.
//!
//! [string]: crate::construct::string
diff --git a/src/construct/partial_whitespace.rs b/src/construct/partial_whitespace.rs
index bf06df9..04016cb 100644
--- a/src/construct/partial_whitespace.rs
+++ b/src/construct/partial_whitespace.rs
@@ -71,7 +71,7 @@ pub fn resolve_whitespace(tokenizer: &mut Tokenizer, hard_break: bool, trim_whol
}
}
-/// Trim a [`Data`][Name::Data] token.
+/// Trim a [`Data`][Name::Data] event.
fn trim_data(
tokenizer: &mut Tokenizer,
exit_index: usize,
@@ -109,7 +109,7 @@ fn trim_data(
};
// The whole data is whitespace.
- // We can be very fast: we only change the token types.
+ // We can be very fast: we only change the event names.
if index == 0 {
tokenizer.events[exit_index - 1].name = name.clone();
tokenizer.events[exit_index].name = name;
@@ -157,7 +157,7 @@ fn trim_data(
}
// The whole data is whitespace.
- // We can be very fast: we only change the token types.
+ // We can be very fast: we only change the event names.
if index == slice.bytes.len() {
tokenizer.events[exit_index - 1].name = Name::SpaceOrTab;
tokenizer.events[exit_index].name = Name::SpaceOrTab;