diff options
Diffstat (limited to 'src/util')
| -rw-r--r-- | src/util/decode_character_reference.rs | 103 | ||||
| -rw-r--r-- | src/util/encode.rs | 29 | ||||
| -rw-r--r-- | src/util/mod.rs | 6 | ||||
| -rw-r--r-- | src/util/sanitize_uri.rs | 111 | ||||
| -rw-r--r-- | src/util/span.rs | 112 |
5 files changed, 361 insertions, 0 deletions
diff --git a/src/util/decode_character_reference.rs b/src/util/decode_character_reference.rs new file mode 100644 index 0000000..4a9317e --- /dev/null +++ b/src/util/decode_character_reference.rs @@ -0,0 +1,103 @@ +//! Utilities to decode character references. + +use crate::constant::{CHARACTER_REFERENCE_NAMES, CHARACTER_REFERENCE_VALUES}; + +/// Decode named character references. +/// +/// Turn the name coming from a named character reference (without the `&` or +/// `;`) into a string. +/// This looks the given string up in [`CHARACTER_REFERENCE_NAMES`][] and then +/// takes the corresponding value from [`CHARACTER_REFERENCE_VALUES`][]. +/// +/// The result is `String` instead of `char` because named character references +/// can expand into multiple characters. +/// +/// ## Examples +/// +/// ```rust ignore +/// use micromark::util::character_reference::decode_named; +/// +/// assert_eq!(decode_named("amp"), "&"); +/// assert_eq!(decode_named("AElig"), "Æ"); +/// assert_eq!(decode_named("aelig"), "æ"); +/// ``` +/// +/// ## Panics +/// +/// This function panics if a name not in [`CHARACTER_REFERENCE_NAMES`][] is +/// given. +/// It is expected that figuring out whether a name is allowed is handled in +/// the parser. +/// When `micromark` is used, this function never panics. +/// +/// ## References +/// +/// * [`wooorm/decode-named-character-reference`](https://github.com/wooorm/decode-named-character-reference) +/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references) +pub fn decode_named(value: &str) -> String { + let position = CHARACTER_REFERENCE_NAMES.iter().position(|&x| x == value); + if let Some(index) = position { + CHARACTER_REFERENCE_VALUES[index].to_string() + } else { + unreachable!("expected valid `name`") + } +} + +/// Decode numeric character references. +/// +/// Turn the number (in string form as either hexadecimal or decimal) coming +/// from a numeric character reference into a character. +/// Whether the base of the string form is `10` (decimal) or `16` (hexadecimal) +/// must be passed as the `radix` parameter. +/// +/// This returns the `char` associated with that number or a replacement +/// character for C0 control characters (except for ASCII whitespace), C1 +/// control characters, lone surrogates, noncharacters, and out of range +/// characters. +/// +/// ## Examples +/// +/// ```rust ignore +/// use micromark::util::character_reference::decode_numeric; +/// +/// assert_eq!(decode_numeric("123", 10), '{'); +/// assert_eq!(decode_numeric("9", 16), '\t'); +/// assert_eq!(decode_numeric("0", 10), '�'); // Not allowed. +/// ``` +/// +/// ## Panics +/// +/// This function panics if a invalid string or an out of bounds valid string +/// is given. +/// It is expected that figuring out whether a number is allowed is handled in +/// the parser. +/// When `micromark` is used, this function never panics. +/// +/// ## References +/// +/// * [`micromark-util-decode-numeric-character-reference` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-decode-numeric-character-reference) +/// * [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references) +pub fn decode_numeric(value: &str, radix: u32) -> char { + let code = u32::from_str_radix(value, radix).expect("expected `value` to be an int"); + + if + // C0 except for HT, LF, FF, CR, space + code < 0x09 || + code == 0x0B || + (code > 0x0D && code < 0x20) || + // Control character (DEL) of the basic block and C1 controls. + (code > 0x7E && code < 0xA0) || + // Lone high surrogates and low surrogates. + (code > 0xd7ff && code < 0xe000) || + // Noncharacters. + (code > 0xfdcf && code < 0xfdf0) || + ((code & 0xffff) == 0xffff) || + ((code & 0xffff) == 0xfffe) || + // Out of range + code > 0x0010_ffff + { + '�' + } else { + char::from_u32(code).expect("expected valid `code`") + } +} diff --git a/src/util/encode.rs b/src/util/encode.rs new file mode 100644 index 0000000..f79c8ea --- /dev/null +++ b/src/util/encode.rs @@ -0,0 +1,29 @@ +//! Utilities to encode HTML. + +/// Encode dangerous html characters. +/// +/// This ensures that certain characters which have special meaning in HTML are +/// dealt with. +/// Technically, we can skip `>` and `"` in many cases, but CM includes them. +/// +/// This behavior is not explained in prose in `CommonMark` but can be inferred +/// from the input/output test cases. +/// +/// ## Examples +/// +/// ```rust ignore +/// use micromark::util::encode; +/// +/// assert_eq!(encode("I <3 🦀"), "I <3 🦀"); +/// ``` +/// +/// ## References +/// +/// * [`micromark-util-encode` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-encode) +pub fn encode(value: &str) -> String { + value + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") +} diff --git a/src/util/mod.rs b/src/util/mod.rs new file mode 100644 index 0000000..c3db267 --- /dev/null +++ b/src/util/mod.rs @@ -0,0 +1,6 @@ +//! Utilities used when compiling markdown. + +pub mod decode_character_reference; +pub mod encode; +pub mod sanitize_uri; +pub mod span; diff --git a/src/util/sanitize_uri.rs b/src/util/sanitize_uri.rs new file mode 100644 index 0000000..1dffc50 --- /dev/null +++ b/src/util/sanitize_uri.rs @@ -0,0 +1,111 @@ +//! Utilities to make urls safe. + +use crate::util::encode::encode; + +/// Make a value safe for injection as a URL. +/// +/// This encodes unsafe characters with percent-encoding and skips already +/// encoded sequences (see `normalize_uri` below). +/// Further unsafe characters are encoded as character references (see +/// `encode`). +/// +/// Then, a vec of (lowercase) allowed protocols can be given, in which case +/// the URL is sanitized. +/// +/// For example, `Some(vec!["http", "https", "irc", "ircs", "mailto", "xmpp"])` +/// can be used for `a[href]`, or `Some(vec!["http", "https"])` for `img[src]`. +/// If the URL includes an unknown protocol (one not matched by `protocol`, such +/// as a dangerous example, `javascript:`), the value is ignored. +/// +/// ## References +/// +/// * [`micromark-util-sanitize-uri` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-sanitize-uri) +pub fn sanitize_uri(value: &str, protocols: &Option<Vec<&str>>) -> String { + let value = encode(&normalize_uri(value)); + + if let Some(protocols) = protocols { + let chars: Vec<char> = value.chars().collect(); + let mut index = 0; + let mut colon: Option<usize> = None; + + while index < chars.len() { + let char = chars[index]; + + match char { + ':' => { + colon = Some(index); + break; + } + '?' | '#' | '/' => break, + _ => {} + } + + index += 1; + } + + // If there is no protocol, or the first colon is after `?`, `#`, or `/`, it’s relative. + // It is a protocol, it should be allowed. + if let Some(colon) = colon { + let protocol = chars[0..colon].iter().collect::<String>().to_lowercase(); + if !protocols.contains(&protocol.as_str()) { + return "".to_string(); + } + } + } + + value +} + +/// Normalize a URL (such as used in definitions). +/// +/// Encode unsafe characters with percent-encoding, skipping already encoded +/// sequences. +/// +/// ## References +/// +/// * [`micromark-util-sanitize-uri` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-sanitize-uri) +fn normalize_uri(value: &str) -> String { + let chars: Vec<char> = value.chars().collect(); + let mut result: Vec<String> = vec![]; + let mut index = 0; + let mut start = 0; + let mut buff = [0; 4]; + + while index < chars.len() { + let char = chars[index]; + + // A correct percent encoded value. + if char == '%' + && index + 2 < chars.len() + && chars[index + 1].is_ascii_alphanumeric() + && chars[index + 2].is_ascii_alphanumeric() + { + index += 3; + continue; + } + + // Note: Rust already takes care of lone astral surrogates. + // Non-ascii or not allowed ascii. + if char >= '\u{0080}' + || !matches!(char, '!' | '#' | '$' | '&'..=';' | '=' | '?'..='Z' | '_' | 'a'..='z' | '~') + { + result.push(chars[start..index].iter().collect::<String>()); + + char.encode_utf8(&mut buff); + result.push( + buff[0..char.len_utf8()] + .iter() + .map(|&byte| format!("%{:X}", byte)) + .collect::<String>(), + ); + + start = index + 1; + } + + index += 1; + } + + result.push(chars[start..].iter().collect::<String>()); + + result.join("") +} diff --git a/src/util/span.rs b/src/util/span.rs new file mode 100644 index 0000000..c48549b --- /dev/null +++ b/src/util/span.rs @@ -0,0 +1,112 @@ +//! Utilities to deal with semantic labels. + +use crate::tokenizer::{Code, Event, EventType}; + +/// A struct representing the span of an opening and closing event of a token. +#[derive(Debug)] +pub struct Span { + // To do: probably needed in the future. + // start: Point, + /// Absolute offset (and `index` in `codes`) of where this span starts. + pub start_index: usize, + // To do: probably needed in the future. + // end: Point, + /// Absolute offset (and `index` in `codes`) of where this span ends. + pub end_index: usize, + // To do: probably needed in the future. + // token_type: TokenType, +} + +/// Get a span from an event. +/// +/// Get the span of an `exit` event, by looking backwards through the events to +/// find the corresponding `enter` event. +/// This assumes that tokens with the same are not nested. +/// +/// ## Panics +/// +/// This function panics if an enter event is given. +/// When `micromark` is used, this function never panics. +pub fn from_exit_event(events: &[Event], index: usize) -> Span { + let exit = &events[index]; + // let end = exit.point.clone(); + let end_index = exit.index; + let token_type = exit.token_type.clone(); + // To do: support `enter` events if needed and walk forwards? + assert_eq!( + exit.event_type, + EventType::Exit, + "expected `get_span` to be called on `exit` event" + ); + let mut enter_index = index - 1; + + loop { + let enter = &events[enter_index]; + if enter.event_type == EventType::Enter && enter.token_type == token_type { + return Span { + // start: enter.point.clone(), + start_index: enter.index, + // end, + end_index, + // token_type, + }; + } + + enter_index -= 1; + } +} + +/// Serialize a span, optionally expanding tabs. +pub fn serialize(all_codes: &[Code], span: &Span, expand_tabs: bool) -> String { + serialize_codes(codes(all_codes, span), expand_tabs) +} + +/// Get a slice of codes from a span. +pub fn codes<'a>(codes: &'a [Code], span: &Span) -> &'a [Code] { + &codes[span.start_index..span.end_index] +} + +/// Serialize a slice of codes, optionally expanding tabs. +fn serialize_codes(codes: &[Code], expand_tabs: bool) -> String { + let mut at_tab = false; + let mut index = 0; + let mut value: Vec<char> = vec![]; + + while index < codes.len() { + let code = codes[index]; + let mut at_tab_next = false; + + match code { + Code::CarriageReturnLineFeed => { + value.push('\r'); + value.push('\n'); + } + Code::Char(char) if char == '\n' || char == '\r' => { + value.push(char); + } + Code::Char(char) if char == '\t' => { + at_tab_next = true; + value.push(if expand_tabs { ' ' } else { char }); + } + Code::VirtualSpace => { + if !expand_tabs && at_tab { + index += 1; + continue; + } + value.push(' '); + } + Code::Char(char) => { + value.push(char); + } + Code::None => { + unreachable!("unexpected EOF code in codes"); + } + } + + at_tab = at_tab_next; + + index += 1; + } + + value.into_iter().collect() +} |