From cfbeb05e32914ed951b7ce4afd131ef75b3cfb32 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Thu, 30 Jan 2025 01:46:04 +0100 Subject: Fix code block merging with previous spans in `markdown` widget --- widget/src/markdown.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index c0648e9e..fe61d631 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -344,7 +344,16 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { })); } - None + let prev = if spans.is_empty() { + None + } else { + produce( + &mut lists, + Item::Paragraph(Text::new(spans.drain(..).collect())), + ) + }; + + prev } pulldown_cmark::Tag::MetadataBlock(_) => { metadata = true; -- cgit From d49d4dc3fa58482ab0269ac678134fa6f360396a Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Thu, 30 Jan 2025 01:46:52 +0100 Subject: Make `spacing` configurable in `markdown::Settings` --- widget/src/markdown.rs | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index fe61d631..252a3e1a 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -527,6 +527,8 @@ pub struct Settings { pub h6_size: Pixels, /// The text size used in code blocks. pub code_size: Pixels, + /// The spacing to be used between elements. + pub spacing: Pixels, } impl Settings { @@ -547,6 +549,7 @@ impl Settings { h5_size: text_size, h6_size: text_size, code_size: text_size * 0.75, + spacing: text_size * 0.875, } } } @@ -649,10 +652,9 @@ where h5_size, h6_size, code_size, + spacing, } = settings; - let spacing = text_size * 0.625; - let blocks = items.into_iter().enumerate().map(|(i, item)| match item { Item::Heading(level, heading) => { container(rich_text(heading.spans(style)).size(match level { @@ -675,11 +677,21 @@ where } Item::List { start: None, items } => { column(items.iter().map(|items| { - row![text("•").size(text_size), view(items, settings, style)] - .spacing(spacing) - .into() + row![ + text("•").size(text_size), + view( + items, + Settings { + spacing: settings.spacing * 0.6, + ..settings + }, + style + ) + ] + .spacing(spacing) + .into() })) - .spacing(spacing) + .spacing(spacing * 0.75) .into() } Item::List { @@ -688,12 +700,19 @@ where } => column(items.iter().enumerate().map(|(i, items)| { row![ text!("{}.", i as u64 + *start).size(text_size), - view(items, settings, style) + view( + items, + Settings { + spacing: settings.spacing * 0.6, + ..settings + }, + style + ) ] .spacing(spacing) .into() })) - .spacing(spacing) + .spacing(spacing * 0.75) .into(), Item::CodeBlock(code) => container( scrollable( -- cgit From aa0f0e73aa06242b8228fd81df8acaac6f377b71 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Thu, 30 Jan 2025 01:47:10 +0100 Subject: Let `markdown::view` be `Shrink` when no code blocks exist --- widget/src/markdown.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 252a3e1a..2b7bc0fc 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -735,7 +735,7 @@ where .into(), }); - Element::new(column(blocks).width(Length::Fill).spacing(text_size)) + Element::new(column(blocks).spacing(spacing)) } /// The theme catalog of Markdown items. -- cgit From 128058ea948909c21a9cfd0b58cbd3a13e238e57 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Fri, 31 Jan 2025 17:35:38 +0100 Subject: Draft incremental `markdown` parsing Specially useful when dealing with long Markdown streams, like LLMs. --- widget/src/markdown.rs | 99 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 17 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 2b7bc0fc..0365dee8 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -47,6 +47,7 @@ //! } //! } //! ``` +#![allow(missing_docs)] use crate::core::border; use crate::core::font::{self, Font}; use crate::core::padding; @@ -57,12 +58,47 @@ use crate::core::{ use crate::{column, container, rich_text, row, scrollable, span, text}; use std::cell::{Cell, RefCell}; +use std::ops::Range; use std::sync::Arc; pub use core::text::Highlight; pub use pulldown_cmark::HeadingLevel; pub use url::Url; +#[derive(Debug, Clone)] +pub struct Content { + items: Vec, + state: State, +} + +impl Content { + pub fn parse(markdown: &str) -> Self { + let mut state = State::default(); + let items = parse_with(&mut state, markdown).collect(); + + Self { items, state } + } + + pub fn push_str(&mut self, markdown: &str) { + // Append to last leftover text + let mut leftover = std::mem::take(&mut self.state.leftover); + leftover.push_str(markdown); + + // Pop the last item + let _ = self.items.pop(); + + // Re-parse last item and new text + let new_items = parse_with(&mut self.state, &leftover); + self.items.extend(new_items); + + dbg!(&self.state); + } + + pub fn items(&self) -> &[Item] { + &self.items + } +} + /// A Markdown item. #[derive(Debug, Clone)] pub enum Item { @@ -232,6 +268,24 @@ impl Span { /// } /// ``` pub fn parse(markdown: &str) -> impl Iterator + '_ { + parse_with(State::default(), markdown) +} + +#[derive(Debug, Clone, Default)] +pub struct State { + leftover: String, +} + +impl AsMut for State { + fn as_mut(&mut self) -> &mut Self { + self + } +} + +fn parse_with<'a>( + mut state: impl AsMut + 'a, + markdown: &'a str, +) -> impl Iterator + 'a { struct List { start: Option, items: Vec>, @@ -255,27 +309,31 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { | pulldown_cmark::Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS | pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH, - ); - - let produce = |lists: &mut Vec, item| { - if lists.is_empty() { - Some(item) - } else { - lists - .last_mut() - .expect("list context") - .items - .last_mut() - .expect("item context") - .push(item); + ) + .into_offset_iter(); - None - } - }; + let mut produce = + move |lists: &mut Vec, item, source: Range| { + if lists.is_empty() { + state.as_mut().leftover = markdown[source.start..].to_owned(); + + Some(item) + } else { + lists + .last_mut() + .expect("list context") + .items + .last_mut() + .expect("item context") + .push(item); + + None + } + }; // We want to keep the `spans` capacity #[allow(clippy::drain_collect)] - parser.filter_map(move |event| match event { + parser.filter_map(move |(event, source)| match event { pulldown_cmark::Event::Start(tag) => match tag { pulldown_cmark::Tag::Strong if !metadata && !table => { strong = true; @@ -311,6 +369,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), + source, ) }; @@ -350,6 +409,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), + source, ) }; @@ -370,6 +430,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::Heading(level, Text::new(spans.drain(..).collect())), + source, ) } pulldown_cmark::TagEnd::Strong if !metadata && !table => { @@ -392,6 +453,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), + source, ) } pulldown_cmark::TagEnd::Item if !metadata && !table => { @@ -401,6 +463,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), + source, ) } } @@ -413,6 +476,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { start: list.start, items: list.items, }, + source, ) } pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => { @@ -424,6 +488,7 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { produce( &mut lists, Item::CodeBlock(Text::new(spans.drain(..).collect())), + source, ) } pulldown_cmark::TagEnd::MetadataBlock(_) => { -- cgit From 4b8fc23840e52a81f1c62c48e4e83d04b700b392 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Fri, 31 Jan 2025 20:37:07 +0100 Subject: Implement `markdown` incremental code highlighting --- widget/src/markdown.rs | 175 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 129 insertions(+), 46 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 0365dee8..7f6965e5 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -57,6 +57,7 @@ use crate::core::{ }; use crate::{column, container, rich_text, row, scrollable, span, text}; +use std::borrow::BorrowMut; use std::cell::{Cell, RefCell}; use std::ops::Range; use std::sync::Arc; @@ -65,7 +66,7 @@ pub use core::text::Highlight; pub use pulldown_cmark::HeadingLevel; pub use url::Url; -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Content { items: Vec, state: State, @@ -80,6 +81,10 @@ impl Content { } pub fn push_str(&mut self, markdown: &str) { + if markdown.is_empty() { + return; + } + // Append to last leftover text let mut leftover = std::mem::take(&mut self.state.leftover); leftover.push_str(markdown); @@ -90,8 +95,6 @@ impl Content { // Re-parse last item and new text let new_items = parse_with(&mut self.state, &leftover); self.items.extend(new_items); - - dbg!(&self.state); } pub fn items(&self) -> &[Item] { @@ -271,19 +274,91 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { parse_with(State::default(), markdown) } -#[derive(Debug, Clone, Default)] -pub struct State { +#[derive(Debug, Default)] +struct State { leftover: String, + #[cfg(feature = "highlighter")] + highlighter: Option, +} + +#[cfg(feature = "highlighter")] +#[derive(Debug)] +struct Highlighter { + lines: Vec<(String, Vec)>, + parser: iced_highlighter::Stream, + current: usize, } -impl AsMut for State { - fn as_mut(&mut self) -> &mut Self { - self +#[cfg(feature = "highlighter")] +impl Highlighter { + pub fn new(language: &str) -> Self { + Self { + lines: Vec::new(), + parser: iced_highlighter::Stream::new( + &iced_highlighter::Settings { + theme: iced_highlighter::Theme::Base16Ocean, + token: language.to_string(), + }, + ), + current: 0, + } + } + + pub fn prepare(&mut self) { + self.current = 0; + } + + pub fn highlight_line(&mut self, text: &str) -> &[Span] { + match self.lines.get(self.current) { + Some(line) if line.0 == text => {} + _ => { + if self.current + 1 < self.lines.len() { + println!("Resetting..."); + self.parser.reset(); + self.lines.truncate(self.current); + + for line in &self.lines { + println!("Refeeding {n} lines", n = self.lines.len()); + + let _ = self.parser.highlight_line(&line.0); + } + } + + println!("Parsing: {text}", text = text.trim_end()); + if self.current + 1 < self.lines.len() { + self.parser.commit(); + } + + let mut spans = Vec::new(); + + for (range, highlight) in self.parser.highlight_line(text) { + spans.push(Span::Highlight { + text: text[range].to_owned(), + color: highlight.color(), + font: highlight.font(), + }); + } + + if self.current + 1 == self.lines.len() { + let _ = self.lines.pop(); + } + + self.lines.push((text.to_owned(), spans)); + } + } + + self.current += 1; + + &self + .lines + .get(self.current - 1) + .expect("Line must be parsed") + .1 } } fn parse_with<'a>( - mut state: impl AsMut + 'a, + mut state: impl BorrowMut + 'a, markdown: &'a str, ) -> impl Iterator + 'a { struct List { @@ -312,24 +387,26 @@ fn parse_with<'a>( ) .into_offset_iter(); - let mut produce = - move |lists: &mut Vec, item, source: Range| { - if lists.is_empty() { - state.as_mut().leftover = markdown[source.start..].to_owned(); - - Some(item) - } else { - lists - .last_mut() - .expect("list context") - .items - .last_mut() - .expect("item context") - .push(item); + let produce = move |state: &mut State, + lists: &mut Vec, + item, + source: Range| { + if lists.is_empty() { + state.leftover = markdown[source.start..].to_owned(); + + Some(item) + } else { + lists + .last_mut() + .expect("list context") + .items + .last_mut() + .expect("item context") + .push(item); - None - } - }; + None + } + }; // We want to keep the `spans` capacity #[allow(clippy::drain_collect)] @@ -367,6 +444,7 @@ fn parse_with<'a>( None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -393,20 +471,24 @@ fn parse_with<'a>( ) if !metadata && !table => { #[cfg(feature = "highlighter")] { - use iced_highlighter::Highlighter; - use text::Highlighter as _; - - highlighter = - Some(Highlighter::new(&iced_highlighter::Settings { - theme: iced_highlighter::Theme::Base16Ocean, - token: _language.to_string(), - })); + highlighter = Some({ + let mut highlighter = state + .borrow_mut() + .highlighter + .take() + .unwrap_or_else(|| Highlighter::new(&_language)); + + highlighter.prepare(); + + highlighter + }); } let prev = if spans.is_empty() { None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -428,6 +510,7 @@ fn parse_with<'a>( pulldown_cmark::Event::End(tag) => match tag { pulldown_cmark::TagEnd::Heading(level) if !metadata && !table => { produce( + state.borrow_mut(), &mut lists, Item::Heading(level, Text::new(spans.drain(..).collect())), source, @@ -451,6 +534,7 @@ fn parse_with<'a>( } pulldown_cmark::TagEnd::Paragraph if !metadata && !table => { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -461,6 +545,7 @@ fn parse_with<'a>( None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -471,6 +556,7 @@ fn parse_with<'a>( let list = lists.pop().expect("list context"); produce( + state.borrow_mut(), &mut lists, Item::List { start: list.start, @@ -482,10 +568,11 @@ fn parse_with<'a>( pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => { #[cfg(feature = "highlighter")] { - highlighter = None; + state.borrow_mut().highlighter = highlighter.take(); } produce( + state.borrow_mut(), &mut lists, Item::CodeBlock(Text::new(spans.drain(..).collect())), source, @@ -504,20 +591,16 @@ fn parse_with<'a>( pulldown_cmark::Event::Text(text) if !metadata && !table => { #[cfg(feature = "highlighter")] if let Some(highlighter) = &mut highlighter { - use text::Highlighter as _; + let start = std::time::Instant::now(); - for (range, highlight) in - highlighter.highlight_line(text.as_ref()) - { - let span = Span::Highlight { - text: text[range].to_owned(), - color: highlight.color(), - font: highlight.font(), - }; - - spans.push(span); + for line in text.lines() { + spans.extend_from_slice( + highlighter.highlight_line(&format!("{line}\n")), + ); } + dbg!(start.elapsed()); + return None; } -- cgit From bc2d662af7fd9b527dc6b49f31627780e58d79c2 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Fri, 31 Jan 2025 20:42:53 +0100 Subject: Replace `println` with `log` calls in `markdown` module --- widget/src/markdown.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 7f6965e5..77a560ec 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -313,18 +313,22 @@ impl Highlighter { Some(line) if line.0 == text => {} _ => { if self.current + 1 < self.lines.len() { - println!("Resetting..."); + log::debug!("Resetting highlighter..."); self.parser.reset(); self.lines.truncate(self.current); for line in &self.lines { - println!("Refeeding {n} lines", n = self.lines.len()); + log::debug!( + "Refeeding {n} lines", + n = self.lines.len() + ); let _ = self.parser.highlight_line(&line.0); } } - println!("Parsing: {text}", text = text.trim_end()); + log::trace!("Parsing: {text}", text = text.trim_end()); + if self.current + 1 < self.lines.len() { self.parser.commit(); } -- cgit From 095859ed57e573d91ebe36dceb888ec95427b6ca Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Fri, 31 Jan 2025 20:49:25 +0100 Subject: Add `new` constructor for `markdown::Content` --- widget/src/markdown.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 77a560ec..b4b89095 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -66,13 +66,17 @@ pub use core::text::Highlight; pub use pulldown_cmark::HeadingLevel; pub use url::Url; -#[derive(Debug)] +#[derive(Debug, Default)] pub struct Content { items: Vec, state: State, } impl Content { + pub fn new() -> Self { + Self::default() + } + pub fn parse(markdown: &str) -> Self { let mut state = State::default(); let items = parse_with(&mut state, markdown).collect(); @@ -595,16 +599,12 @@ fn parse_with<'a>( pulldown_cmark::Event::Text(text) if !metadata && !table => { #[cfg(feature = "highlighter")] if let Some(highlighter) = &mut highlighter { - let start = std::time::Instant::now(); - for line in text.lines() { spans.extend_from_slice( highlighter.highlight_line(&format!("{line}\n")), ); } - dbg!(start.elapsed()); - return None; } -- cgit From 447f5ae494da7ef93ac073600f4e5a2559c4e71c Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Sat, 1 Feb 2025 00:33:05 +0100 Subject: Discard `markdown::Highlighter` if language changes --- widget/src/markdown.rs | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index b4b89095..658166ec 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -289,6 +289,7 @@ struct State { #[derive(Debug)] struct Highlighter { lines: Vec<(String, Vec)>, + language: String, parser: iced_highlighter::Stream, current: usize, } @@ -304,6 +305,7 @@ impl Highlighter { token: language.to_string(), }, ), + language: language.to_owned(), current: 0, } } @@ -484,6 +486,9 @@ fn parse_with<'a>( .borrow_mut() .highlighter .take() + .filter(|highlighter| { + highlighter.language == _language.as_ref() + }) .unwrap_or_else(|| Highlighter::new(&_language)); highlighter.prepare(); -- cgit From eb81679e604e2d1d45590c236fb5b2644c38f3d5 Mon Sep 17 00:00:00 2001 From: Héctor Ramón Jiménez Date: Sat, 1 Feb 2025 01:07:35 +0100 Subject: Split code blocks into multiple `rich_text` lines This improves layout diffing considerably! --- widget/src/markdown.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'widget/src/markdown.rs') diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 658166ec..bb818d19 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -116,7 +116,7 @@ pub enum Item { /// A code block. /// /// You can enable the `highlighter` feature for syntax highlighting. - CodeBlock(Text), + CodeBlock(Vec), /// A list. List { /// The first number of the list, if it is ordered. @@ -377,6 +377,7 @@ fn parse_with<'a>( } let mut spans = Vec::new(); + let mut code = Vec::new(); let mut strong = false; let mut emphasis = false; let mut strikethrough = false; @@ -587,7 +588,7 @@ fn parse_with<'a>( produce( state.borrow_mut(), &mut lists, - Item::CodeBlock(Text::new(spans.drain(..).collect())), + Item::CodeBlock(code.drain(..).collect()), source, ) } @@ -605,9 +606,9 @@ fn parse_with<'a>( #[cfg(feature = "highlighter")] if let Some(highlighter) = &mut highlighter { for line in text.lines() { - spans.extend_from_slice( - highlighter.highlight_line(&format!("{line}\n")), - ); + code.push(Text::new( + highlighter.highlight_line(line).to_vec(), + )); } return None; @@ -871,13 +872,14 @@ where })) .spacing(spacing * 0.75) .into(), - Item::CodeBlock(code) => container( + Item::CodeBlock(lines) => container( scrollable( - container( - rich_text(code.spans(style)) + container(column(lines.iter().map(|line| { + rich_text(line.spans(style)) .font(Font::MONOSPACE) - .size(code_size), - ) + .size(code_size) + .into() + }))) .padding(spacing.0 / 2.0), ) .direction(scrollable::Direction::Horizontal( -- cgit