From a1ce43e428754084474a7ecf88ae6debf88b9164 Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Tue, 14 Jun 2022 13:47:32 +0200
Subject: Reorganize to split util
---
readme.md | 2 +-
src/compiler.rs | 44 +++--
src/construct/character_reference.rs | 4 +-
src/construct/code_fenced.rs | 6 +-
src/construct/mod.rs | 1 -
src/content/flow.rs | 4 +-
src/subtokenize.rs | 10 +-
src/tokenizer.rs | 2 +
src/util.rs | 341 ---------------------------------
src/util/decode_character_reference.rs | 103 ++++++++++
src/util/encode.rs | 29 +++
src/util/mod.rs | 6 +
src/util/sanitize_uri.rs | 111 +++++++++++
src/util/span.rs | 112 +++++++++++
14 files changed, 399 insertions(+), 376 deletions(-)
delete mode 100644 src/util.rs
create mode 100644 src/util/decode_character_reference.rs
create mode 100644 src/util/encode.rs
create mode 100644 src/util/mod.rs
create mode 100644 src/util/sanitize_uri.rs
create mode 100644 src/util/span.rs
diff --git a/readme.md b/readme.md
index a735a36..17458ab 100644
--- a/readme.md
+++ b/readme.md
@@ -72,7 +72,6 @@ cargo doc --document-private-items
- [ ] (1) Add docs to subtokenize
- [ ] (1) Add module docs to content
- [ ] (1) Add module docs to parser
-- [ ] (1) Reorganize to split util
- [ ] (1) Add examples to `CompileOptions` docs
- [ ] (1) Add overview docs on how everything works
- [ ] (1) Move safe protocols to constants
@@ -165,6 +164,7 @@ cargo doc --document-private-items
- [x] (1) Optionally remove dangerous protocols when compiling
- [x] (1) Add docs to html (text)
- [x] (1) Add docs on bnf
+- [x] (1) Reorganize to split util
### Extensions
diff --git a/src/compiler.rs b/src/compiler.rs
index 619bbe5..e1ce440 100644
--- a/src/compiler.rs
+++ b/src/compiler.rs
@@ -2,8 +2,10 @@
use crate::construct::character_reference::Kind as CharacterReferenceKind;
use crate::tokenizer::{Code, Event, EventType, TokenType};
use crate::util::{
- decode_named_character_reference, decode_numeric_character_reference, encode, get_span,
- sanitize_uri, slice_serialize,
+ decode_character_reference::{decode_named, decode_numeric},
+ encode::encode,
+ sanitize_uri::sanitize_uri,
+ span::{from_exit_event, serialize},
};
/// Configuration (optional).
@@ -141,7 +143,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
ignore_encode = false;
}
TokenType::HtmlFlowData | TokenType::HtmlTextData => {
- let slice = slice_serialize(codes, &get_span(events, index), false);
+ let slice = serialize(codes, &from_exit_event(events, index), false);
let res = if ignore_encode { slice } else { encode(&slice) };
@@ -208,9 +210,9 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
}
TokenType::CodeFlowChunk => {
code_flow_seen_data = Some(true);
- buf_tail_mut(buffers).push(encode(&slice_serialize(
+ buf_tail_mut(buffers).push(encode(&serialize(
codes,
- &get_span(events, index),
+ &from_exit_event(events, index),
false,
)));
}
@@ -224,13 +226,17 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
if let Some(buf) = atx_heading_buffer {
atx_heading_buffer = Some(
buf.to_string()
- + &encode(&slice_serialize(codes, &get_span(events, index), false)),
+ + &encode(&serialize(
+ codes,
+ &from_exit_event(events, index),
+ false,
+ )),
);
}
// First fence we see.
if None == atx_opening_sequence_size {
- let rank = slice_serialize(codes, &get_span(events, index), false).len();
+ let rank = serialize(codes, &from_exit_event(events, index), false).len();
atx_opening_sequence_size = Some(rank);
buf_tail_mut(buffers).push(format!("<h{}>", rank));
}
@@ -246,7 +252,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
atx_heading_buffer = Some("".to_string());
}
- let slice = encode(&slice_serialize(codes, &get_span(events, index), false));
+ let slice = encode(&serialize(codes, &from_exit_event(events, index), false));
println!("slice: {:?}", slice);
buf_tail_mut(buffers).push(slice);
}
@@ -258,7 +264,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
atx_heading_buffer = None;
}
TokenType::AutolinkProtocol => {
- let slice = slice_serialize(codes, &get_span(events, index), false);
+ let slice = serialize(codes, &from_exit_event(events, index), false);
let buf = buf_tail_mut(buffers);
buf.push(format!(
"<a href=\"{}\">",
@@ -268,7 +274,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
buf.push("</a>".to_string());
}
TokenType::AutolinkEmail => {
- let slice = slice_serialize(codes, &get_span(events, index), false);
+ let slice = serialize(codes, &from_exit_event(events, index), false);
let buf = buf_tail_mut(buffers);
buf.push(format!(
"<a href=\"mailto:{}\">",
@@ -289,9 +295,9 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
// } else if code_text_inside {
// buf_tail_mut(buffers).push(" ".to_string());
} else {
- buf_tail_mut(buffers).push(encode(&slice_serialize(
+ buf_tail_mut(buffers).push(encode(&serialize(
codes,
- &get_span(events, index),
+ &from_exit_event(events, index),
false,
)));
}
@@ -308,18 +314,16 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
TokenType::CharacterReferenceValue => {
let kind = character_reference_kind
.expect("expected `character_reference_kind` to be set");
- let reference = slice_serialize(codes, &get_span(events, index), false);
+ let reference = serialize(codes, &from_exit_event(events, index), false);
let ref_string = reference.as_str();
let value = match kind {
CharacterReferenceKind::Decimal => {
- decode_numeric_character_reference(ref_string, 10).to_string()
+ decode_numeric(ref_string, 10).to_string()
}
CharacterReferenceKind::Hexadecimal => {
- decode_numeric_character_reference(ref_string, 16).to_string()
- }
- CharacterReferenceKind::Named => {
- decode_named_character_reference(ref_string)
+ decode_numeric(ref_string, 16).to_string()
}
+ CharacterReferenceKind::Named => decode_named(ref_string),
};
buf_tail_mut(buffers).push(value);
@@ -329,9 +333,9 @@ pub fn compile(events: &[Event], codes: &[Code], options: &CompileOptions) -> St
// This branch below currently acts as the resulting `data` tokens.
TokenType::Data | TokenType::CharacterEscapeValue => {
// last_was_tag = false;
- buf_tail_mut(buffers).push(encode(&slice_serialize(
+ buf_tail_mut(buffers).push(encode(&serialize(
codes,
- &get_span(events, index),
+ &from_exit_event(events, index),
false,
)));
}
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index 943d70f..ba2ad61 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -20,7 +20,7 @@
//! However, for security reasons, several numeric character references parse
//! fine but are not rendered as their corresponding character and they are
//! instead replaced by a U+FFFD REPLACEMENT CHARACTER (`�`).
-//! See [`decode_numeric_character_reference`][decode_numeric] for more info.
+//! See [`decode_numeric`][decode_numeric] for more info.
//!
//! To escape ASCII punctuation characters, use the terser
//! [character escape][character_escape] construct instead (as in, `\&`).
@@ -48,7 +48,7 @@
//! [string]: crate::content::string
//! [text]: crate::content::text
//! [character_escape]: crate::construct::character_reference
-//! [decode_numeric]: crate::util::decode_numeric_character_reference
+//! [decode_numeric]: crate::util::decode_character_reference::decode_numeric
//! [character_reference_names]: crate::constant::CHARACTER_REFERENCE_NAMES
//! [html]: https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state
//!
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index a440d40..c852e8d 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -91,7 +91,7 @@
use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE};
use crate::construct::partial_whitespace::start as whitespace;
use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
-use crate::util::get_span;
+use crate::util::span::from_exit_event;
/// Kind of fences.
#[derive(Debug, Clone, PartialEq)]
@@ -147,7 +147,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
if let Some(event) = tail {
if event.token_type == TokenType::Whitespace {
- let span = get_span(&tokenizer.events, tokenizer.events.len() - 1);
+ let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1);
prefix = span.end_index - span.start_index;
}
}
@@ -415,7 +415,7 @@ fn close_sequence_before(tokenizer: &mut Tokenizer, info: Info, code: Code) -> S
if let Some(event) = tail {
if event.token_type == TokenType::Whitespace {
- let span = get_span(&tokenizer.events, tokenizer.events.len() - 1);
+ let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1);
prefix = span.end_index - span.start_index;
}
}
diff --git a/src/construct/mod.rs b/src/construct/mod.rs
index d2203d2..14f53a0 100644
--- a/src/construct/mod.rs
+++ b/src/construct/mod.rs
@@ -53,7 +53,6 @@
//! example `ascii_punctuation` refers to
//! [`char::is_ascii_punctuation`][char::is_ascii_punctuation].
-
pub mod autolink;
pub mod blank_line;
pub mod character_escape;
diff --git a/src/content/flow.rs b/src/content/flow.rs
index 6fa8c25..4d2ece1 100644
--- a/src/content/flow.rs
+++ b/src/content/flow.rs
@@ -28,7 +28,7 @@ use crate::construct::{
};
use crate::subtokenize::subtokenize;
use crate::tokenizer::{Code, Event, Point, State, StateFnResult, TokenType, Tokenizer};
-use crate::util::get_span;
+use crate::util::span::from_exit_event;
/// Turn `codes` as the flow content type into events.
pub fn flow(codes: &[Code], point: Point, index: usize) -> Vec<Event> {
@@ -240,7 +240,7 @@ fn continuation_construct_after_prefix(tokenizer: &mut Tokenizer, code: Code) ->
if let Some(event) = tail {
if event.token_type == TokenType::Whitespace {
- let span = get_span(&tokenizer.events, tokenizer.events.len() - 1);
+ let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1);
prefix = span.end_index - span.start_index;
}
}
diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index e004f7b..35d7672 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -1,10 +1,8 @@
-use crate::content::content::start as content;
-use crate::content::string::start as string;
-use crate::content::text::start as text;
+use crate::content::{content::start as content, string::start as string, text::start as text};
use crate::tokenizer::{
Code, Event, EventType, State, StateFn, StateFnResult, TokenType, Tokenizer,
};
-use crate::util::{slice_codes, Span};
+use crate::util::span;
use std::collections::HashMap;
/// To do.
@@ -51,7 +49,7 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) {
while let Some(index_ptr) = index_opt {
let enter = &events[index_ptr];
assert_eq!(enter.event_type, EventType::Enter);
- let span = Span {
+ let span = span::Span {
start_index: enter.index,
end_index: events[index_ptr + 1].index,
};
@@ -66,7 +64,7 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) {
_ => unreachable!("cannot be ok/nok"),
};
- result = tokenizer.feed(slice_codes(codes, &span), func, enter.next == None);
+ result = tokenizer.feed(span::codes(codes, &span), func, enter.next == None);
if let Some(ref x) = result.1 {
if !x.is_empty() {
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 8a2f477..486bc75 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -513,6 +513,8 @@ impl Tokenizer {
let mut state = State::Fn(Box::new(start));
let mut index = 0;
+ println!("feed: {:?} {:?}", codes, drain);
+
self.consumed = true;
while index < codes.len() {
diff --git a/src/util.rs b/src/util.rs
deleted file mode 100644
index accc48e..0000000
--- a/src/util.rs
+++ /dev/null
@@ -1,341 +0,0 @@
-//! Some utilities helpful when parsing and compiling markdown.
-
-use crate::constant::{CHARACTER_REFERENCE_NAMES, CHARACTER_REFERENCE_VALUES};
-use crate::tokenizer::{Code, Event, EventType};
-
-/// Encode dangerous html characters.
-///
-/// This ensures that certain characters which have special meaning in HTML are
-/// dealt with.
-/// Technically, we can skip `>` and `"` in many cases, but CM includes them.
-///
-/// This behavior is not explained in prose in `CommonMark` but can be inferred
-/// from the input/output test cases.
-///
-/// ## Examples
-///
-/// ```rust ignore
-/// use micromark::util::encode;
-///
-/// assert_eq!(encode("I <3 🦀"), "I <3 🦀");
-/// ```
-///
-/// ## References
-///
-/// * [`micromark-util-encode` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-encode)
-pub fn encode(value: &str) -> String {
- value
- .replace('&', "&")
- .replace('"', """)
- .replace('<', "<")
- .replace('>', ">")
-}
-
-/// Make a value safe for injection as a URL.
-///
-/// This encodes unsafe characters with percent-encoding and skips already
-/// encoded sequences (see `normalize_uri` below).
-/// Further unsafe characters are encoded as character references (see
-/// `encode`).
-///
-/// Then, a vec of (lowercase) allowed protocols can be given, in which case
-/// the URL is sanitized.
-///
-/// For example, `Some(vec!["http", "https", "irc", "ircs", "mailto", "xmpp"])`
-/// can be used for `a[href]`, or `Some(vec!["http", "https"])` for `img[src]`.
-/// If the URL includes an unknown protocol (one not matched by `protocol`, such
-/// as a dangerous example, `javascript:`), the value is ignored.
-pub fn sanitize_uri(value: &str, protocols: &Option<Vec<&str>>) -> String {
- let value = encode(&normalize_uri(value));
-
- if let Some(protocols) = protocols {
- let chars: Vec<char> = value.chars().collect();
- let mut index = 0;
- let mut colon: Option<usize> = None;
-
- while index < chars.len() {
- let char = chars[index];
-
- match char {
- ':' => {
- colon = Some(index);
- break;
- }
- '?' | '#' | '/' => break,
- _ => {}
- }
-
- index += 1;
- }
-
- // If there is no protocol, or the first colon is after `?`, `#`, or `/`, it’s relative.
- // It is a protocol, it should be allowed.
- if let Some(colon) = colon {
- let protocol = chars[0..colon].iter().collect::<String>().to_lowercase();
- if !protocols.contains(&protocol.as_str()) {
- return "".to_string();
- }
- }
- }
-
- value
-}
-
-/// Normalize a URL (such as used in definitions).
-///
-/// Encode unsafe characters with percent-encoding, skipping already encoded
-/// sequences.
-fn normalize_uri(value: &str) -> String {
- let chars: Vec<char> = value.chars().collect();
- let mut result: Vec<String> = vec![];
- let mut index = 0;
- let mut start = 0;
- let mut buff = [0; 4];
-
- while index < chars.len() {
- let char = chars[index];
-
- // A correct percent encoded value.
- if char == '%'
- && index + 2 < chars.len()
- && chars[index + 1].is_ascii_alphanumeric()
- && chars[index + 2].is_ascii_alphanumeric()
- {
- index += 3;
- continue;
- }
-
- // Note: Rust already takes care of lone astral surrogates.
- // Non-ascii or not allowed ascii.
- if char >= '\u{0080}'
- || !matches!(char, '!' | '#' | '$' | '&'..=';' | '=' | '?'..='Z' | '_' | 'a'..='z' | '~')
- {
- result.push(chars[start..index].iter().collect::<String>());
-
- char.encode_utf8(&mut buff);
- result.push(
- buff[0..char.len_utf8()]
- .iter()
- .map(|&byte| format!("%{:X}", byte))
- .collect::<String>(),
- );
-
- start = index + 1;
- }
-
- index += 1;
- }
-
- result.push(chars[start..].iter().collect::<String>());
-
- result.join("")
-}
-
-/// Decode numeric character references.
-///
-/// Turn the number (in string form as either hexadecimal or decimal) coming
-/// from a numeric character reference into a character.
-/// Whether the base of the string form is `10` (decimal) or `16` (hexadecimal)
-/// must be passed as the `radix` parameter.
-///
-/// This returns the `char` associated with that number or a replacement
-/// character for C0 control characters (except for ASCII whitespace), C1
-/// control characters, lone surrogates, noncharacters, and out of range
-/// characters.
-///
-/// ## Examples
-///
-/// ```rust ignore
-/// use micromark::util::decode_numeric_character_reference;
-///
-/// assert_eq!(decode_numeric_character_reference("123", 10), '{');
-/// assert_eq!(decode_numeric_character_reference("9", 16), '\t');
-/// assert_eq!(decode_numeric_character_reference("0", 10), '�'); // Not allowed.
-/// ```
-///
-/// ## Panics
-///
-/// This function panics if a invalid string or an out of bounds valid string
-/// is given.
-/// It is expected that figuring out whether a number is allowed is handled in
-/// the parser.
-/// When `micromark` is used, this function never panics.
-///
-/// ## References
-///
-/// * [`micromark-util-decode-numeric-character-reference` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-decode-numeric-character-reference)
-/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
-pub fn decode_numeric_character_reference(value: &str, radix: u32) -> char {
- let code = u32::from_str_radix(value, radix).expect("expected `value` to be an int");
-
- if
- // C0 except for HT, LF, FF, CR, space
- code < 0x09 ||
- code == 0x0B ||
- (code > 0x0D && code < 0x20) ||
- // Control character (DEL) of the basic block and C1 controls.
- (code > 0x7E && code < 0xA0) ||
- // Lone high surrogates and low surrogates.
- (code > 0xd7ff && code < 0xe000) ||
- // Noncharacters.
- (code > 0xfdcf && code < 0xfdf0) ||
- ((code & 0xffff) == 0xffff) ||
- ((code & 0xffff) == 0xfffe) ||
- // Out of range
- code > 0x0010_ffff
- {
- '�'
- } else {
- char::from_u32(code).expect("expected valid `code`")
- }
-}
-
-/// Decode named character references.
-///
-/// Turn the name coming from a named character reference (without the `&` or
-/// `;`) into a string.
-/// This looks the given string up in [`CHARACTER_REFERENCE_NAMES`][] and then
-/// takes the corresponding value from [`CHARACTER_REFERENCE_VALUES`][].
-///
-/// The result is `String` instead of `char` because named character references
-/// can expand into multiple characters.
-///
-/// ## Examples
-///
-/// ```rust ignore
-/// use micromark::util::decode_named_character_reference;
-///
-/// assert_eq!(decode_named_character_reference("amp"), "&");
-/// assert_eq!(decode_named_character_reference("AElig"), "Æ");
-/// assert_eq!(decode_named_character_reference("aelig"), "æ");
-/// ```
-///
-/// ## Panics
-///
-/// This function panics if a name not in [`CHARACTER_REFERENCE_NAMES`][] is
-/// given.
-/// It is expected that figuring out whether a name is allowed is handled in
-/// the parser.
-/// When `micromark` is used, this function never panics.
-///
-/// ## References
-///
-/// * [`wooorm/decode-named-character-reference`](https://github.com/wooorm/decode-named-character-reference)
-/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
-pub fn decode_named_character_reference(value: &str) -> String {
- let position = CHARACTER_REFERENCE_NAMES.iter().position(|&x| x == value);
- if let Some(index) = position {
- CHARACTER_REFERENCE_VALUES[index].to_string()
- } else {
- unreachable!("expected valid `name`")
- }
-}
-
-/// A struct representing the span of an opening and closing event of a token.
-#[derive(Debug)]
-pub struct Span {
- // To do: probably needed in the future.
- // start: Point,
- /// Absolute offset (and `index` in `codes`) of where this span starts.
- pub start_index: usize,
- // To do: probably needed in the future.
- // end: Point,
- /// Absolute offset (and `index` in `codes`) of where this span ends.
- pub end_index: usize,
- // To do: probably needed in the future.
- // token_type: TokenType,
-}
-
-/// Get a span from an event.
-///
-/// Get the span of an `exit` event, by looking backwards through the events to
-/// find the corresponding `enter` event.
-/// This assumes that tokens with the same are not nested.
-///
-/// ## Panics
-///
-/// This function panics if an enter event is given.
-/// When `micromark` is used, this function never panics.
-pub fn get_span(events: &[Event], index: usize) -> Span {
- let exit = &events[index];
- // let end = exit.point.clone();
- let end_index = exit.index;
- let token_type = exit.token_type.clone();
- // To do: support `enter` events if needed and walk forwards?
- assert_eq!(
- exit.event_type,
- EventType::Exit,
- "expected `get_span` to be called on `exit` event"
- );
- let mut enter_index = index - 1;
-
- loop {
- let enter = &events[enter_index];
- if enter.event_type == EventType::Enter && enter.token_type == token_type {
- return Span {
- // start: enter.point.clone(),
- start_index: enter.index,
- // end,
- end_index,
- // token_type,
- };
- }
-
- enter_index -= 1;
- }
-}
-
-/// Serialize a span, optionally expanding tabs.
-pub fn slice_serialize(codes: &[Code], span: &Span, expand_tabs: bool) -> String {
- serialize_chunks(slice_codes(codes, span), expand_tabs)
-}
-
-/// Get a slice of codes from a span.
-pub fn slice_codes<'a>(codes: &'a [Code], span: &Span) -> &'a [Code] {
- &codes[span.start_index..span.end_index]
-}
-
-/// Serialize a slice of codes, optionally expanding tabs.
-pub fn serialize_chunks(codes: &[Code], expand_tabs: bool) -> String {
- let mut at_tab = false;
- let mut index = 0;
- let mut value: Vec<char> = vec![];
-
- while index < codes.len() {
- let code = codes[index];
- let mut at_tab_next = false;
-
- match code {
- Code::CarriageReturnLineFeed => {
- value.push('\r');
- value.push('\n');
- }
- Code::Char(char) if char == '\n' || char == '\r' => {
- value.push(char);
- }
- Code::Char(char) if char == '\t' => {
- at_tab_next = true;
- value.push(if expand_tabs { ' ' } else { char });
- }
- Code::VirtualSpace => {
- if !expand_tabs && at_tab {
- index += 1;
- continue;
- }
- value.push(' ');
- }
- Code::Char(char) => {
- value.push(char);
- }
- Code::None => {
- unreachable!("unexpected EOF code in codes");
- }
- }
-
- at_tab = at_tab_next;
-
- index += 1;
- }
-
- value.into_iter().collect()
-}
diff --git a/src/util/decode_character_reference.rs b/src/util/decode_character_reference.rs
new file mode 100644
index 0000000..4a9317e
--- /dev/null
+++ b/src/util/decode_character_reference.rs
@@ -0,0 +1,103 @@
+//! Utilities to decode character references.
+
+use crate::constant::{CHARACTER_REFERENCE_NAMES, CHARACTER_REFERENCE_VALUES};
+
+/// Decode named character references.
+///
+/// Turn the name coming from a named character reference (without the `&` or
+/// `;`) into a string.
+/// This looks the given string up in [`CHARACTER_REFERENCE_NAMES`][] and then
+/// takes the corresponding value from [`CHARACTER_REFERENCE_VALUES`][].
+///
+/// The result is `String` instead of `char` because named character references
+/// can expand into multiple characters.
+///
+/// ## Examples
+///
+/// ```rust ignore
+/// use micromark::util::character_reference::decode_named;
+///
+/// assert_eq!(decode_named("amp"), "&");
+/// assert_eq!(decode_named("AElig"), "Æ");
+/// assert_eq!(decode_named("aelig"), "æ");
+/// ```
+///
+/// ## Panics
+///
+/// This function panics if a name not in [`CHARACTER_REFERENCE_NAMES`][] is
+/// given.
+/// It is expected that figuring out whether a name is allowed is handled in
+/// the parser.
+/// When `micromark` is used, this function never panics.
+///
+/// ## References
+///
+/// * [`wooorm/decode-named-character-reference`](https://github.com/wooorm/decode-named-character-reference)
+/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
+pub fn decode_named(value: &str) -> String {
+ let position = CHARACTER_REFERENCE_NAMES.iter().position(|&x| x == value);
+ if let Some(index) = position {
+ CHARACTER_REFERENCE_VALUES[index].to_string()
+ } else {
+ unreachable!("expected valid `name`")
+ }
+}
+
+/// Decode numeric character references.
+///
+/// Turn the number (in string form as either hexadecimal or decimal) coming
+/// from a numeric character reference into a character.
+/// Whether the base of the string form is `10` (decimal) or `16` (hexadecimal)
+/// must be passed as the `radix` parameter.
+///
+/// This returns the `char` associated with that number or a replacement
+/// character for C0 control characters (except for ASCII whitespace), C1
+/// control characters, lone surrogates, noncharacters, and out of range
+/// characters.
+///
+/// ## Examples
+///
+/// ```rust ignore
+/// use micromark::util::character_reference::decode_numeric;
+///
+/// assert_eq!(decode_numeric("123", 10), '{');
+/// assert_eq!(decode_numeric("9", 16), '\t');
+/// assert_eq!(decode_numeric("0", 10), '�'); // Not allowed.
+/// ```
+///
+/// ## Panics
+///
+/// This function panics if a invalid string or an out of bounds valid string
+/// is given.
+/// It is expected that figuring out whether a number is allowed is handled in
+/// the parser.
+/// When `micromark` is used, this function never panics.
+///
+/// ## References
+///
+/// * [`micromark-util-decode-numeric-character-reference` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-decode-numeric-character-reference)
+/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
+pub fn decode_numeric(value: &str, radix: u32) -> char {
+ let code = u32::from_str_radix(value, radix).expect("expected `value` to be an int");
+
+ if
+ // C0 except for HT, LF, FF, CR, space
+ code < 0x09 ||
+ code == 0x0B ||
+ (code > 0x0D && code < 0x20) ||
+ // Control character (DEL) of the basic block and C1 controls.
+ (code > 0x7E && code < 0xA0) ||
+ // Lone high surrogates and low surrogates.
+ (code > 0xd7ff && code < 0xe000) ||
+ // Noncharacters.
+ (code > 0xfdcf && code < 0xfdf0) ||
+ ((code & 0xffff) == 0xffff) ||
+ ((code & 0xffff) == 0xfffe) ||
+ // Out of range
+ code > 0x0010_ffff
+ {
+ '�'
+ } else {
+ char::from_u32(code).expect("expected valid `code`")
+ }
+}
diff --git a/src/util/encode.rs b/src/util/encode.rs
new file mode 100644
index 0000000..f79c8ea
--- /dev/null
+++ b/src/util/encode.rs
@@ -0,0 +1,29 @@
+//! Utilities to encode HTML.
+
+/// Encode dangerous html characters.
+///
+/// This ensures that certain characters which have special meaning in HTML are
+/// dealt with.
+/// Technically, we can skip `>` and `"` in many cases, but CM includes them.
+///
+/// This behavior is not explained in prose in `CommonMark` but can be inferred
+/// from the input/output test cases.
+///
+/// ## Examples
+///
+/// ```rust ignore
+/// use micromark::util::encode;
+///
+/// assert_eq!(encode("I <3 🦀"), "I <3 🦀");
+/// ```
+///
+/// ## References
+///
+/// * [`micromark-util-encode` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-encode)
+pub fn encode(value: &str) -> String {
+ value
+ .replace('&', "&")
+ .replace('"', """)
+ .replace('<', "<")
+ .replace('>', ">")
+}
diff --git a/src/util/mod.rs b/src/util/mod.rs
new file mode 100644
index 0000000..c3db267
--- /dev/null
+++ b/src/util/mod.rs
@@ -0,0 +1,6 @@
+//! Utilities used when compiling markdown.
+
+pub mod decode_character_reference;
+pub mod encode;
+pub mod sanitize_uri;
+pub mod span;
diff --git a/src/util/sanitize_uri.rs b/src/util/sanitize_uri.rs
new file mode 100644
index 0000000..1dffc50
--- /dev/null
+++ b/src/util/sanitize_uri.rs
@@ -0,0 +1,111 @@
+//! Utilities to make urls safe.
+
+use crate::util::encode::encode;
+
+/// Make a value safe for injection as a URL.
+///
+/// This encodes unsafe characters with percent-encoding and skips already
+/// encoded sequences (see `normalize_uri` below).
+/// Further unsafe characters are encoded as character references (see
+/// `encode`).
+///
+/// Then, a vec of (lowercase) allowed protocols can be given, in which case
+/// the URL is sanitized.
+///
+/// For example, `Some(vec!["http", "https", "irc", "ircs", "mailto", "xmpp"])`
+/// can be used for `a[href]`, or `Some(vec!["http", "https"])` for `img[src]`.
+/// If the URL includes an unknown protocol (one not matched by `protocol`, such
+/// as a dangerous example, `javascript:`), the value is ignored.
+///
+/// ## References
+///
+/// * [`micromark-util-sanitize-uri` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-sanitize-uri)
+pub fn sanitize_uri(value: &str, protocols: &Option<Vec<&str>>) -> String {
+ let value = encode(&normalize_uri(value));
+
+ if let Some(protocols) = protocols {
+ let chars: Vec<char> = value.chars().collect();
+ let mut index = 0;
+ let mut colon: Option<usize> = None;
+
+ while index < chars.len() {
+ let char = chars[index];
+
+ match char {
+ ':' => {
+ colon = Some(index);
+ break;
+ }
+ '?' | '#' | '/' => break,
+ _ => {}
+ }
+
+ index += 1;
+ }
+
+ // If there is no protocol, or the first colon is after `?`, `#`, or `/`, it’s relative.
+ // It is a protocol, it should be allowed.
+ if let Some(colon) = colon {
+ let protocol = chars[0..colon].iter().collect::<String>().to_lowercase();
+ if !protocols.contains(&protocol.as_str()) {
+ return "".to_string();
+ }
+ }
+ }
+
+ value
+}
+
+/// Normalize a URL (such as used in definitions).
+///
+/// Encode unsafe characters with percent-encoding, skipping already encoded
+/// sequences.
+///
+/// ## References
+///
+/// * [`micromark-util-sanitize-uri` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-sanitize-uri)
+fn normalize_uri(value: &str) -> String {
+ let chars: Vec<char> = value.chars().collect();
+ let mut result: Vec<String> = vec![];
+ let mut index = 0;
+ let mut start = 0;
+ let mut buff = [0; 4];
+
+ while index < chars.len() {
+ let char = chars[index];
+
+ // A correct percent encoded value.
+ if char == '%'
+ && index + 2 < chars.len()
+ && chars[index + 1].is_ascii_alphanumeric()
+ && chars[index + 2].is_ascii_alphanumeric()
+ {
+ index += 3;
+ continue;
+ }
+
+ // Note: Rust already takes care of lone astral surrogates.
+ // Non-ascii or not allowed ascii.
+ if char >= '\u{0080}'
+ || !matches!(char, '!' | '#' | '$' | '&'..=';' | '=' | '?'..='Z' | '_' | 'a'..='z' | '~')
+ {
+ result.push(chars[start..index].iter().collect::<String>());
+
+ char.encode_utf8(&mut buff);
+ result.push(
+ buff[0..char.len_utf8()]
+ .iter()
+ .map(|&byte| format!("%{:X}", byte))
+ .collect::<String>(),
+ );
+
+ start = index + 1;
+ }
+
+ index += 1;
+ }
+
+ result.push(chars[start..].iter().collect::<String>());
+
+ result.join("")
+}
diff --git a/src/util/span.rs b/src/util/span.rs
new file mode 100644
index 0000000..c48549b
--- /dev/null
+++ b/src/util/span.rs
@@ -0,0 +1,112 @@
+//! Utilities to deal with semantic labels.
+
+use crate::tokenizer::{Code, Event, EventType};
+
+/// A struct representing the span of an opening and closing event of a token.
+#[derive(Debug)]
+pub struct Span {
+ // To do: probably needed in the future.
+ // start: Point,
+ /// Absolute offset (and `index` in `codes`) of where this span starts.
+ pub start_index: usize,
+ // To do: probably needed in the future.
+ // end: Point,
+ /// Absolute offset (and `index` in `codes`) of where this span ends.
+ pub end_index: usize,
+ // To do: probably needed in the future.
+ // token_type: TokenType,
+}
+
+/// Get a span from an event.
+///
+/// Get the span of an `exit` event, by looking backwards through the events to
+/// find the corresponding `enter` event.
+/// This assumes that tokens with the same are not nested.
+///
+/// ## Panics
+///
+/// This function panics if an enter event is given.
+/// When `micromark` is used, this function never panics.
+pub fn from_exit_event(events: &[Event], index: usize) -> Span {
+ let exit = &events[index];
+ // let end = exit.point.clone();
+ let end_index = exit.index;
+ let token_type = exit.token_type.clone();
+ // To do: support `enter` events if needed and walk forwards?
+ assert_eq!(
+ exit.event_type,
+ EventType::Exit,
+ "expected `get_span` to be called on `exit` event"
+ );
+ let mut enter_index = index - 1;
+
+ loop {
+ let enter = &events[enter_index];
+ if enter.event_type == EventType::Enter && enter.token_type == token_type {
+ return Span {
+ // start: enter.point.clone(),
+ start_index: enter.index,
+ // end,
+ end_index,
+ // token_type,
+ };
+ }
+
+ enter_index -= 1;
+ }
+}
+
+/// Serialize a span, optionally expanding tabs.
+pub fn serialize(all_codes: &[Code], span: &Span, expand_tabs: bool) -> String {
+ serialize_codes(codes(all_codes, span), expand_tabs)
+}
+
+/// Get a slice of codes from a span.
+pub fn codes<'a>(codes: &'a [Code], span: &Span) -> &'a [Code] {
+ &codes[span.start_index..span.end_index]
+}
+
+/// Serialize a slice of codes, optionally expanding tabs.
+fn serialize_codes(codes: &[Code], expand_tabs: bool) -> String {
+ let mut at_tab = false;
+ let mut index = 0;
+ let mut value: Vec<char> = vec![];
+
+ while index < codes.len() {
+ let code = codes[index];
+ let mut at_tab_next = false;
+
+ match code {
+ Code::CarriageReturnLineFeed => {
+ value.push('\r');
+ value.push('\n');
+ }
+ Code::Char(char) if char == '\n' || char == '\r' => {
+ value.push(char);
+ }
+ Code::Char(char) if char == '\t' => {
+ at_tab_next = true;
+ value.push(if expand_tabs { ' ' } else { char });
+ }
+ Code::VirtualSpace => {
+ if !expand_tabs && at_tab {
+ index += 1;
+ continue;
+ }
+ value.push(' ');
+ }
+ Code::Char(char) => {
+ value.push(char);
+ }
+ Code::None => {
+ unreachable!("unexpected EOF code in codes");
+ }
+ }
+
+ at_tab = at_tab_next;
+
+ index += 1;
+ }
+
+ value.into_iter().collect()
+}
--
cgit